[llvm] [GlobalISel] Migrate remaining generic wip_match_opcode combines to MIR-pattern. (PR #222851)
Vikash Gupta via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 23:20:25 PDT 2026
https://github.com/vg0204 created https://github.com/llvm/llvm-project/pull/222851
This patch converts another batch of GlobalISel combine rules from hand-written C++ / wip_match_opcode matchers to declarative MIR patterns, preserving existing behavior.
- commute_constant_to_rhs, fold_binop_into_select, constant_fold_binop, propagate_undef_all_ops, insert_extract_vec_elt_out_of_bounds, indexed load/store, buildvector_identity_fold
- i2p_to_p2i, ptr_add_with_zero, merge_of_x_and_{undef,zero}
- funnel_shift_to_rotate, combine_minmax_nan, mulo_by_2 / mulo_by_0, adde_to_addo, match_addos / match_subo_no_overflow
- redundant_neg_operands, sub_add_reg, count_zero_to_zero_poison, avgfloor/avgceil, select_constant_cmp splat arms
Functional changes:
- extract_vector_element_with_build_vector{,_trunc}: bail on an out-of-bounds constant index instead of reading past the G_BUILD_VECTOR sources; the undef result is already handled by extract_vec_elt_out_of_bounds.
- AMDGPU s64 and/or with a 32-bit mask: do not narrow when both operands are constants, so constant_fold_binop folds the binop to a single constant instead.
>From 87ea5842a33dbee6885c224a78d21041069c4eed Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Fri, 11 Sep 2026 11:38:16 +0530
Subject: [PATCH] [GlobalISel] Migrate remaining generic wip_match_opcode
combines to MIR-pattern.
This patch converts another batch of GlobalISel combine rules from
hand-written C++ / wip_match_opcode matchers to declarative MIR
patterns, preserving existing behavior.
- commute_constant_to_rhs, fold_binop_into_select, constant_fold_binop,
propagate_undef_all_ops, insert_extract_vec_elt_out_of_bounds,
indexed load/store, buildvector_identity_fold
- i2p_to_p2i, ptr_add_with_zero, merge_of_x_and_{undef,zero}
- funnel_shift_to_rotate, combine_minmax_nan, mulo_by_2 / mulo_by_0,
adde_to_addo, match_addos / match_subo_no_overflow
- redundant_neg_operands, sub_add_reg, count_zero_to_zero_poison,
avgfloor/avgceil, select_constant_cmp splat arms
Functional changes:
- extract_vector_element_with_build_vector{,_trunc}: bail on an
out-of-bounds constant index instead of reading past the
G_BUILD_VECTOR sources; the undef result is already handled by
extract_vec_elt_out_of_bounds.
- AMDGPU s64 and/or with a 32-bit mask: do not narrow when both operands
are constants, so constant_fold_binop folds the binop to a single
constant instead.
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 92 +--
.../include/llvm/Target/GlobalISel/Combine.td | 630 +++++++++++++-----
llvm/lib/CodeGen/GlobalISel/CMakeLists.txt | 1 -
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 360 +---------
.../GlobalISel/CombinerHelperArtifacts.cpp | 85 ---
.../GlobalISel/CombinerHelperVectorOps.cpp | 27 +-
llvm/lib/Target/AMDGPU/AMDGPUCombine.td | 3 +-
.../Target/AMDGPU/AMDGPUCombinerHelper.cpp | 11 +
llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h | 6 +
.../GlobalISel/combine-adde-to-addo.mir | 155 +++++
.../GlobalISel/combine-mulo-with-2.mir | 2 +-
.../combine-shift-immed-mismatch-crash.mir | 8 +-
.../postlegalizer-combiner-undef.mir | 4 +-
.../prelegalizer-combiner-addo-zero.mir | 2 +-
.../prelegalizer-combiner-mulo-zero.mir | 2 +-
.../prelegalizercombiner-prop-extends-phi.mir | 4 +-
.../AArch64/neon-shuffle-vector-tbl.ll | 2 +-
.../combine-binop-s64-with-s32-mask.mir | 6 +-
.../combine-extract-vector-load.mir | 10 +-
.../combine-fold-binop-into-select.mir | 4 +-
.../GlobalISel/combine-foldable-fneg.mir | 4 +-
.../AMDGPU/GlobalISel/combine-fsub-fneg.mir | 14 +-
...mbine-shl-from-extend-narrow.postlegal.mir | 8 +-
.../llvm.amdgcn.make.buffer.rsrc.ll | 4 +-
...ctlz-from-umul-to-lshr-in-postlegalizer.ll | 45 +-
.../llvm/lib/CodeGen/GlobalISel/BUILD.gn | 1 -
26 files changed, 729 insertions(+), 761 deletions(-)
delete mode 100644 llvm/lib/CodeGen/GlobalISel/CombinerHelperArtifacts.cpp
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/combine-adde-to-addo.mir
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 6b80367d786a5..f287f066291ca 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -406,10 +406,6 @@ class CombinerHelper {
LLVM_ABI void applyCombineShlOfExtend(MachineInstr &MI,
const RegisterImmPair &MatchData) const;
- /// Fold away a merge of an unmerge of the corresponding values.
- LLVM_ABI bool matchCombineMergeUnmerge(MachineInstr &MI,
- Register &MatchInfo) const;
-
/// Reduce a shift by a constant to an unmerge and a shift on a half sized
/// type. This will not produce a shift smaller than \p TargetShiftSize.
LLVM_ABI bool matchCombineShiftToUnmerge(MachineInstr &MI,
@@ -455,24 +451,6 @@ class CombinerHelper {
LLVM_ABI bool matchConstantFoldUnaryIntOp(MachineInstr &MI,
BuildFnTy &MatchInfo) const;
- /// Transform PtrToInt(IntToPtr(x)) to x.
- LLVM_ABI void applyCombineP2IToI2P(MachineInstr &MI, Register &Reg) const;
-
- /// Transform G_ADD (G_PTRTOINT x), y -> G_PTRTOINT (G_PTR_ADD x, y)
- /// Transform G_ADD y, (G_PTRTOINT x) -> G_PTRTOINT (G_PTR_ADD x, y)
- LLVM_ABI bool
- matchCombineAddP2IToPtrAdd(MachineInstr &MI,
- std::pair<Register, bool> &PtrRegAndCommute) const;
- LLVM_ABI void
- applyCombineAddP2IToPtrAdd(MachineInstr &MI,
- std::pair<Register, bool> &PtrRegAndCommute) const;
-
- // Transform G_PTR_ADD (G_PTRTOINT C1), C2 -> C1 + C2
- LLVM_ABI bool matchCombineConstPtrAddToI2P(MachineInstr &MI,
- APInt &NewCst) const;
- LLVM_ABI void applyCombineConstPtrAddToI2P(MachineInstr &MI,
- APInt &NewCst) const;
-
/// Transform anyext(trunc(x)) to x.
LLVM_ABI bool matchCombineAnyExtTrunc(MachineInstr &MI, Register &Reg) const;
@@ -500,19 +478,9 @@ class CombinerHelper {
/// Return true if a G_SHUFFLE_VECTOR instruction \p MI has an undef mask.
LLVM_ABI bool matchUndefShuffleVectorMask(MachineInstr &MI) const;
- /// Return true if a G_STORE instruction \p MI is storing an undef value.
- LLVM_ABI bool matchUndefStore(MachineInstr &MI) const;
-
- /// Return true if a G_SELECT instruction \p MI has an undef comparison.
- LLVM_ABI bool matchUndefSelectCmp(MachineInstr &MI) const;
-
/// Return true if a G_{EXTRACT,INSERT}_VECTOR_ELT has an out of range index.
LLVM_ABI bool matchInsertExtractVecEltOutOfBounds(MachineInstr &MI) const;
- /// Return true if a G_SELECT instruction \p MI has a constant comparison. If
- /// true, \p OpIdx will store the operand index of the known selected value.
- LLVM_ABI bool matchConstantSelectCmp(MachineInstr &MI, unsigned &OpIdx) const;
-
/// Replace an instruction with a G_FCONSTANT with value \p C.
LLVM_ABI void replaceInstWithFConstant(MachineInstr &MI, double C) const;
@@ -575,14 +543,6 @@ class CombinerHelper {
/// Erase \p MI
LLVM_ABI void eraseInst(MachineInstr &MI) const;
- /// Return true if MI is a G_ADD which can be simplified to a G_SUB.
- LLVM_ABI bool
- matchSimplifyAddToSub(MachineInstr &MI,
- std::tuple<Register, Register> &MatchInfo) const;
- LLVM_ABI void
- applySimplifyAddToSub(MachineInstr &MI,
- std::tuple<Register, Register> &MatchInfo) const;
-
/// Fold `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`
LLVM_ABI bool matchBinopWithNeg(MachineInstr &MI, BuildFnTy &MatchInfo) const;
@@ -644,8 +604,8 @@ class CombinerHelper {
std::pair<Register, Register> &MatchInfo) const;
///}
- /// Combine G_PTR_ADD with nullptr to G_INTTOPTR
- LLVM_ABI bool matchPtrAddZero(MachineInstr &MI) const;
+ /// Combine a vector G_PTR_ADD with a zero splat to G_INTTOPTR.
+ LLVM_ABI bool matchPtrAddZeroVector(MachineInstr &MI) const;
/// Combine G_UREM x, (known power of 2) to an add and bitmasking.
LLVM_ABI void applySimplifyURemByPow2(MachineInstr &MI) const;
@@ -707,8 +667,6 @@ class CombinerHelper {
LLVM_ABI bool matchOrShiftToFunnelShift(MachineInstr &MI,
bool AllowScalarConstants,
BuildFnTy &MatchInfo) const;
- LLVM_ABI bool matchFunnelShiftToRotate(MachineInstr &MI) const;
- LLVM_ABI void applyFunnelShiftToRotate(MachineInstr &MI) const;
LLVM_ABI bool matchRotateOutOfRange(MachineInstr &MI) const;
LLVM_ABI void applyRotateOutOfRange(MachineInstr &MI) const;
@@ -846,32 +804,11 @@ class CombinerHelper {
LLVM_ABI bool matchTruncUSatUToFPTOUISat(MachineInstr &MI,
MachineInstr &SrcMI) const;
- /// Match:
- /// (G_UMULO x, 2) -> (G_UADDO x, x)
- /// (G_SMULO x, 2) -> (G_SADDO x, x)
- LLVM_ABI bool matchMulOBy2(MachineInstr &MI, BuildFnTy &MatchInfo) const;
-
/// Match:
/// (G_*MULO x, 0) -> 0 + no carry out
LLVM_ABI bool matchMulOBy0(MachineInstr &MI, BuildFnTy &MatchInfo) const;
- /// Match:
- /// (G_*ADDE x, y, 0) -> (G_*ADDO x, y)
- /// (G_*SUBE x, y, 0) -> (G_*SUBO x, y)
- LLVM_ABI bool matchAddEToAddO(MachineInstr &MI, BuildFnTy &MatchInfo) const;
-
- /// Transform (fadd x, fneg(y)) -> (fsub x, y)
- /// (fadd fneg(x), y) -> (fsub y, x)
- /// (fsub x, fneg(y)) -> (fadd x, y)
- /// (fmul fneg(x), fneg(y)) -> (fmul x, y)
- /// (fdiv fneg(x), fneg(y)) -> (fdiv x, y)
- /// (fmad fneg(x), fneg(y), z) -> (fmad x, y, z)
- /// (fma fneg(x), fneg(y), z) -> (fma x, y, z)
- LLVM_ABI bool matchRedundantNegOperands(MachineInstr &MI,
- BuildFnTy &MatchInfo) const;
-
- LLVM_ABI bool matchFsubToFneg(MachineInstr &MI, Register &MatchInfo) const;
- LLVM_ABI void applyFsubToFneg(MachineInstr &MI, Register &MatchInfo) const;
+ LLVM_ABI bool matchFsubToFneg(MachineInstr &MI) const;
LLVM_ABI bool canCombineFMadOrFMA(MachineInstr &MI, bool &AllowFusionGlobally,
bool &HasFMAD, bool &Aggressive,
@@ -927,8 +864,6 @@ class CombinerHelper {
matchCombineFSubFpExtFNegFMulToFMadOrFMA(MachineInstr &MI,
BuildFnTy &MatchInfo) const;
- LLVM_ABI bool matchCombineFMinMaxNaN(MachineInstr &MI, unsigned &Info) const;
-
LLVM_ABI bool
matchRepeatedFPDivisor(MachineInstr &MI,
SmallVector<MachineInstr *> &MatchInfo) const;
@@ -992,9 +927,6 @@ class CombinerHelper {
/// Match constant LHS FP ops that should be commuted.
LLVM_ABI bool matchCommuteFPConstantToRHS(MachineInstr &MI) const;
- // Given a binop \p MI, commute operands 1 and 2.
- LLVM_ABI void applyCommuteBinOpOperands(MachineInstr &MI) const;
-
/// Combine select to integer min/max.
LLVM_ABI bool matchSelectIMinMax(const MachineOperand &MO,
BuildFnTy &MatchInfo) const;
@@ -1024,8 +956,7 @@ class CombinerHelper {
LLVM_ABI bool matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo) const;
/// Combine extract vector element.
- LLVM_ABI bool matchExtractVectorElement(MachineInstr &MI,
- BuildFnTy &MatchInfo) const;
+ LLVM_ABI bool matchExtractVectorElement(MachineInstr &MI) const;
/// Combine extract vector element with a build vector on the vector register.
LLVM_ABI bool
@@ -1074,8 +1005,7 @@ class CombinerHelper {
LLVM_ABI void applyExpandFPowI(MachineInstr &MI, int64_t Exponent) const;
/// Combine insert vector element OOB.
- LLVM_ABI bool matchInsertVectorElementOOB(MachineInstr &MI,
- BuildFnTy &MatchInfo) const;
+ LLVM_ABI bool matchInsertVectorElementOOB(MachineInstr &MI) const;
LLVM_ABI bool
matchFreezeOfSingleMaybePoisonOperand(MachineInstr &MI,
@@ -1134,14 +1064,6 @@ class CombinerHelper {
LLVM_ABI bool matchUnmergeValuesAnyExtBuildVector(const MachineInstr &MI,
BuildFnTy &MatchInfo) const;
- // merge_values(_, undef) -> anyext
- LLVM_ABI bool matchMergeXAndUndef(const MachineInstr &MI,
- BuildFnTy &MatchInfo) const;
-
- // merge_values(_, zero) -> zext
- LLVM_ABI bool matchMergeXAndZero(const MachineInstr &MI,
- BuildFnTy &MatchInfo) const;
-
// overflow sub
LLVM_ABI bool matchSuboCarryOut(const MachineInstr &MI,
BuildFnTy &MatchInfo) const;
@@ -1154,11 +1076,7 @@ class CombinerHelper {
// (ctlz (or (shl (xor x, (sra x, bitwidth-1)), 1), 1) -> (ctls x)
LLVM_ABI bool matchCtls(MachineInstr &CtlzMI, BuildFnTy &MatchInfo) const;
- LLVM_ABI bool matchAVG(MachineInstr &MI, MachineRegisterInfo &MRI, Register X,
- Register Y, unsigned TargetOpc) const;
-
LLVM_ABI bool matchCountZeroToZeroPoison(MachineInstr &MI) const;
- LLVM_ABI void applyCountZeroToZeroPoison(MachineInstr &MI) const;
private:
/// Checks for legality of an indexed variant of \p LdSt.
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 3735e95ee8f68..4c40e8c8c8b38 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -243,11 +243,15 @@ def push_freeze_to_prevent_poison_from_propagating : GICombineRule<
[{ return !isGuaranteedNotToBePoison(${src}.getReg(), MRI) && Helper.matchFreezeOfSingleMaybePoisonOperand(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+def extending_loads_frags : GICombinePatFrag<
+ (outs root:$dst), (ins),
+ !foreach(op, [G_LOAD, G_SEXTLOAD, G_ZEXTLOAD],
+ (pattern (op $dst, $ptr)))>;
def extending_loads : GICombineRule<
(defs root:$root, extending_load_matchdata:$matchinfo),
- (match (wip_match_opcode G_LOAD, G_SEXTLOAD, G_ZEXTLOAD):$root,
- [{ return Helper.matchCombineExtendingLoads(*${root}, ${matchinfo}); }]),
- (apply [{ Helper.applyCombineExtendingLoads(*${root}, ${matchinfo}); }])>;
+ (match (extending_loads_frags $root):$mi,
+ [{ return Helper.matchCombineExtendingLoads(*${mi}, ${matchinfo}); }]),
+ (apply [{ Helper.applyCombineExtendingLoads(*${mi}, ${matchinfo}); }])>;
def load_and_mask : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
@@ -290,11 +294,22 @@ def combine_extracted_vector_load : GICombineRule<
[{ return Helper.matchCombineExtractedVectorLoad(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
-def combine_indexed_load_store : GICombineRule<
+def combine_indexed_load_frags : GICombinePatFrag<
+ (outs root:$dst), (ins),
+ !foreach(op, [G_LOAD, G_SEXTLOAD, G_ZEXTLOAD],
+ (pattern (op $dst, $ptr)))>;
+def combine_indexed_load : GICombineRule<
(defs root:$root, indexed_load_store_matchdata:$matchinfo),
- (match (wip_match_opcode G_LOAD, G_SEXTLOAD, G_ZEXTLOAD, G_STORE):$root,
+ (match (combine_indexed_load_frags $root):$mi,
+ [{ return Helper.matchCombineIndexedLoadStore(*${mi}, ${matchinfo}); }]),
+ (apply [{ Helper.applyCombineIndexedLoadStore(*${mi}, ${matchinfo}); }])>;
+def combine_indexed_store : GICombineRule<
+ (defs root:$root, indexed_load_store_matchdata:$matchinfo),
+ (match (G_STORE $val, $ptr):$root,
[{ return Helper.matchCombineIndexedLoadStore(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyCombineIndexedLoadStore(*${root}, ${matchinfo}); }])>;
+def combine_indexed_load_store : GICombineGroup<[combine_indexed_load,
+ combine_indexed_store]>;
def memcpy_family_matchinfo : GIDefMatchData<"MemCpyFamilyLoweringInfo">;
def combine_memcpy_inline : GICombineRule<
@@ -368,22 +383,29 @@ def shifts_too_big : GICombineRule<
}
}])>;
+// Shared frag for the shift-chain rules below: all shift opcodes that have a
+// (dst, base, amount) shape.
+def shift_chain_frags : GICombinePatFrag<
+ (outs root:$dst), (ins),
+ !foreach(op, [G_SHL, G_ASHR, G_LSHR, G_SSHLSAT, G_USHLSAT],
+ (pattern (op $dst, $base, $amt)))>;
+
// Fold shift (shift base x), y -> shift base, (x+y), if shifts are same
def shift_immed_matchdata : GIDefMatchData<"RegisterImmPair">;
def shift_immed_chain : GICombineRule<
(defs root:$d, shift_immed_matchdata:$matchinfo),
- (match (wip_match_opcode G_SHL, G_ASHR, G_LSHR, G_SSHLSAT, G_USHLSAT):$d,
- [{ return Helper.matchShiftImmedChain(*${d}, ${matchinfo}); }]),
- (apply [{ Helper.applyShiftImmedChain(*${d}, ${matchinfo}); }])>;
+ (match (shift_chain_frags $d):$mi,
+ [{ return Helper.matchShiftImmedChain(*${mi}, ${matchinfo}); }]),
+ (apply [{ Helper.applyShiftImmedChain(*${mi}, ${matchinfo}); }])>;
// Transform shift (logic (shift X, C0), Y), C1
// -> logic (shift X, (C0+C1)), (shift Y, C1), if shifts are same
def shift_of_shifted_logic_matchdata : GIDefMatchData<"ShiftOfShiftedLogic">;
def shift_of_shifted_logic_chain : GICombineRule<
(defs root:$d, shift_of_shifted_logic_matchdata:$matchinfo),
- (match (wip_match_opcode G_SHL, G_ASHR, G_LSHR, G_USHLSAT, G_SSHLSAT):$d,
- [{ return Helper.matchShiftOfShiftedLogic(*${d}, ${matchinfo}); }]),
- (apply [{ Helper.applyShiftOfShiftedLogic(*${d}, ${matchinfo}); }])>;
+ (match (shift_chain_frags $d):$mi,
+ [{ return Helper.matchShiftOfShiftedLogic(*${mi}, ${matchinfo}); }]),
+ (apply [{ Helper.applyShiftOfShiftedLogic(*${mi}, ${matchinfo}); }])>;
def mul_to_shl : GICombineRule<
(defs root:$d, unsigned_matchinfo:$matchinfo),
@@ -537,7 +559,7 @@ def binop_right_undef_to_undef_frags : binop_right_undef_frag<[G_SHL, G_ASHR, G_
def binop_right_undef_to_undef: GICombineRule<
(defs root:$dst),
(match (binop_right_undef_to_undef_frags $dst)),
- (apply [{ Helper.replaceInstWithUndef(*${dst}.getParent()); }])>;
+ (apply (G_IMPLICIT_DEF $dst))>;
def unary_undef_to_zero_frags : unary_undef_frag<[G_ABS]>;
def unary_undef_to_zero: GICombineRule<
@@ -550,7 +572,7 @@ def unary_undef_to_undef_frags : unary_undef_frag<
def unary_undef_to_undef : GICombineRule<
(defs root:$dst),
(match (unary_undef_to_undef_frags $dst)),
- (apply [{ Helper.replaceInstWithUndef(*${dst}.getParent()); }])>;
+ (apply (G_IMPLICIT_DEF $dst))>;
// Instructions where if any source operand is undef, the instruction can be
// replaced with undef.
@@ -558,15 +580,21 @@ def propagate_undef_any_op_frags : binop_any_undef_frag<[G_ADD, G_SUB, G_XOR]>;
def propagate_undef_any_op: GICombineRule<
(defs root:$dst),
(match (propagate_undef_any_op_frags $dst)),
- (apply [{ Helper.replaceInstWithUndef(*${dst}.getParent()); }])>;
+ (apply (G_IMPLICIT_DEF $dst))>;
-// Instructions where if all source operands are undef, the instruction can be
-// replaced with undef.
-def propagate_undef_all_ops: GICombineRule<
+def propagate_undef_all_ops_shuffle_vector: GICombineRule<
(defs root:$root),
- (match (wip_match_opcode G_SHUFFLE_VECTOR, G_BUILD_VECTOR):$root,
- [{ return Helper.matchAllExplicitUsesAreUndef(*${root}); }]),
- (apply [{ Helper.replaceInstWithUndef(*${root}); }])>;
+ (match (G_SHUFFLE_VECTOR $root, $src1, $src2, $mask):$mi,
+ [{ return Helper.matchAllExplicitUsesAreUndef(*${mi}); }]),
+ (apply (G_IMPLICIT_DEF $root))>;
+def propagate_undef_all_ops_build_vector: GICombineRule<
+ (defs root:$root),
+ (match (G_BUILD_VECTOR $root, GIVariadic<>:$srcs):$mi,
+ [{ return Helper.matchAllExplicitUsesAreUndef(*${mi}); }]),
+ (apply (G_IMPLICIT_DEF $root))>;
+
+def propagate_undef_all_ops: GICombineGroup<[propagate_undef_all_ops_shuffle_vector,
+ propagate_undef_all_ops_build_vector]>;
// Replace a G_SHUFFLE_VECTOR with an undef mask with a G_IMPLICIT_DEF.
def propagate_undef_shuffle_mask: GICombineRule<
@@ -575,12 +603,20 @@ def propagate_undef_shuffle_mask: GICombineRule<
[{ return Helper.matchUndefShuffleVectorMask(*${root}); }]),
(apply (G_IMPLICIT_DEF $dst))>;
- // Replace an insert/extract element of an out of bounds index with undef.
- def insert_extract_vec_elt_out_of_bounds : GICombineRule<
+def insert_vec_elt_out_of_bounds : GICombineRule<
(defs root:$root),
- (match (wip_match_opcode G_INSERT_VECTOR_ELT, G_EXTRACT_VECTOR_ELT):$root,
- [{ return Helper.matchInsertExtractVecEltOutOfBounds(*${root}); }]),
- (apply [{ Helper.replaceInstWithUndef(*${root}); }])>;
+ (match (G_INSERT_VECTOR_ELT $root, $vec, $elt, $idx):$mi,
+ [{ return Helper.matchInsertExtractVecEltOutOfBounds(*${mi}); }]),
+ (apply (G_IMPLICIT_DEF $root))>;
+def extract_vec_elt_out_of_bounds : GICombineRule<
+ (defs root:$root),
+ (match (G_EXTRACT_VECTOR_ELT $root, $vec, $idx):$mi,
+ [{ return Helper.matchInsertExtractVecEltOutOfBounds(*${mi}); }]),
+ (apply (G_IMPLICIT_DEF $root))>;
+
+// Replace an insert/extract element of an out of bounds index with undef.
+def insert_extract_vec_elt_out_of_bounds : GICombineGroup<[
+ insert_vec_elt_out_of_bounds, extract_vec_elt_out_of_bounds]>;
// Fold (cond ? x : x) -> x
// _trivial: arms are the same register; _equiv: arms are provably equivalent.
@@ -619,15 +655,26 @@ def select_constant_cmp_true : GICombineRule<
(apply (GIReplaceReg $dst, $tval))
>;
-def select_constant_cmp_general: GICombineRule<
- (defs root:$root, unsigned_matchinfo:$matchinfo),
+def select_constant_cmp_general_false: GICombineRule<
+ (defs root:$dst),
(match (G_SELECT $dst, $cond, $tval, $fval):$root,
- [{ return Helper.matchConstantSelectCmp(*${root}, ${matchinfo}); }]),
- (apply [{ Helper.replaceSingleDefInstWithOperand(*${root}, ${matchinfo}); }])
->;
+ [{
+ auto C = isConstantOrConstantSplatVector(${cond}.getReg(), MRI);
+ return C && C->isZero();
+ }]),
+ (apply (COPY $dst, $fval))>;
+def select_constant_cmp_general_true: GICombineRule<
+ (defs root:$dst),
+ (match (G_SELECT $dst, $cond, $tval, $fval):$root,
+ [{
+ auto C = isConstantOrConstantSplatVector(${cond}.getReg(), MRI);
+ return C && !C->isZero();
+ }]),
+ (apply (COPY $dst, $tval))>;
def select_constant_cmp : GICombineGroup<[select_constant_cmp_false,
select_constant_cmp_true,
- select_constant_cmp_general]>;
+ select_constant_cmp_general_false,
+ select_constant_cmp_general_true]>;
// select c, 0, x -> and (not c), x
// Skip constant c: select_constant_cmp folds it directly, avoiding the freeze.
@@ -665,25 +712,65 @@ def select_not: GICombineRule<
// Fold (C op x) -> (x op C)
// TODO: handle more isCommutable opcodes
// TODO: handle compares (currently not marked as isCommutable)
-def commute_int_constant_to_rhs : GICombineRule<
+// The commutable integer opcodes have three operand shapes: plain 2-use
+// single-def binops, 3-use [su]mulfix ops with a scale operand, and two-def
+// overflow ops.
+class commute_int_binop_rule<Instruction op> : GICombineRule<
(defs root:$root),
- (match (wip_match_opcode G_ADD, G_MUL, G_AND, G_OR, G_XOR,
- G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_UADDO, G_SADDO,
- G_UMULO, G_SMULO, G_UMULH, G_SMULH,
- G_UADDSAT, G_SADDSAT, G_SMULFIX, G_UMULFIX,
- G_SMULFIXSAT, G_UMULFIXSAT):$root,
+ (match (op $dst, $lhs, $rhs):$root,
[{ return Helper.matchCommuteConstantToRHS(*${root}); }]),
- (apply [{ Helper.applyCommuteBinOpOperands(*${root}); }])
->;
+ (apply (op $dst, $rhs, $lhs, (MIFlags $root)))>;
+
+class commute_int_mulfix_rule<Instruction op> : GICombineRule<
+ (defs root:$root),
+ (match (op $dst, $lhs, $rhs, $scale):$root,
+ [{ return Helper.matchCommuteConstantToRHS(*${root}); }]),
+ (apply (op $dst, $rhs, $lhs, $scale, (MIFlags $root)))>;
-def commute_fp_constant_to_rhs : GICombineRule<
+class commute_int_overflow_rule<Instruction op> : GICombineRule<
(defs root:$root),
- (match (wip_match_opcode G_FADD, G_FMUL, G_FMINNUM, G_FMAXNUM,
- G_FMINNUM_IEEE, G_FMAXNUM_IEEE,
- G_FMINIMUM, G_FMAXIMUM):$root,
+ (match (op $dst, $carry, $lhs, $rhs):$root,
+ [{ return Helper.matchCommuteConstantToRHS(*${root}); }]),
+ (apply (op $dst, $carry, $rhs, $lhs, (MIFlags $root)))>;
+
+foreach op = [G_ADD, G_MUL, G_AND, G_OR, G_XOR, G_SMIN, G_SMAX, G_UMIN,
+ G_UMAX, G_UMULH, G_SMULH, G_UADDSAT, G_SADDSAT] in
+ def op#"_commute_constant_to_rhs" : commute_int_binop_rule<op>;
+foreach op = [G_SMULFIX, G_UMULFIX, G_SMULFIXSAT, G_UMULFIXSAT] in
+ def op#"_commute_constant_to_rhs" : commute_int_mulfix_rule<op>;
+foreach op = [G_UADDO, G_SADDO, G_UMULO, G_SMULO] in
+ def op#"_commute_constant_to_rhs" : commute_int_overflow_rule<op>;
+
+def commute_int_constant_to_rhs : GICombineGroup<[
+ G_ADD_commute_constant_to_rhs, G_MUL_commute_constant_to_rhs,
+ G_AND_commute_constant_to_rhs, G_OR_commute_constant_to_rhs,
+ G_XOR_commute_constant_to_rhs, G_SMIN_commute_constant_to_rhs,
+ G_SMAX_commute_constant_to_rhs, G_UMIN_commute_constant_to_rhs,
+ G_UMAX_commute_constant_to_rhs, G_UMULH_commute_constant_to_rhs,
+ G_SMULH_commute_constant_to_rhs, G_UADDSAT_commute_constant_to_rhs,
+ G_SADDSAT_commute_constant_to_rhs,
+ G_SMULFIX_commute_constant_to_rhs, G_UMULFIX_commute_constant_to_rhs,
+ G_SMULFIXSAT_commute_constant_to_rhs, G_UMULFIXSAT_commute_constant_to_rhs,
+ G_UADDO_commute_constant_to_rhs, G_SADDO_commute_constant_to_rhs,
+ G_UMULO_commute_constant_to_rhs, G_SMULO_commute_constant_to_rhs]>;
+
+class commute_fp_binop_rule<Instruction op> : GICombineRule<
+ (defs root:$root),
+ (match (op $dst, $lhs, $rhs):$root,
[{ return Helper.matchCommuteFPConstantToRHS(*${root}); }]),
- (apply [{ Helper.applyCommuteBinOpOperands(*${root}); }])
->;
+ (apply (op $dst, $rhs, $lhs, (MIFlags $root)))>;
+
+foreach op = [G_FADD, G_FMUL, G_FMINNUM, G_FMAXNUM, G_FMINNUM_IEEE,
+ G_FMAXNUM_IEEE, G_FMINIMUM, G_FMAXIMUM] in
+ def op#"_commute_fp_constant_to_rhs" : commute_fp_binop_rule<op>;
+
+def commute_fp_constant_to_rhs : GICombineGroup<[
+ G_FADD_commute_fp_constant_to_rhs, G_FMUL_commute_fp_constant_to_rhs,
+ G_FMINNUM_commute_fp_constant_to_rhs, G_FMAXNUM_commute_fp_constant_to_rhs,
+ G_FMINNUM_IEEE_commute_fp_constant_to_rhs,
+ G_FMAXNUM_IEEE_commute_fp_constant_to_rhs,
+ G_FMINIMUM_commute_fp_constant_to_rhs,
+ G_FMAXIMUM_commute_fp_constant_to_rhs]>;
def commute_constant_to_rhs : GICombineGroup<[
commute_int_constant_to_rhs,
@@ -776,25 +863,31 @@ def same_val_zero: GICombineRule<
(apply (COPY $dst, (GITypeOf<"$x"> 0)))
>;
-// Fold (0 op x) - > 0
+// Fold (0 op x) -> 0. Use matchConstantOp so splat-zero LHS matches.
def binop_left_to_zero_frags : GICombinePatFrag<
- (outs root:$dst, $zero), (ins $rhs),
+ (outs root:$dst), (ins $zero, $rhs),
!foreach(op,
[G_SHL, G_LSHR, G_ASHR, G_SDIV, G_UDIV, G_SREM, G_UREM, G_MUL],
- (pattern (G_CONSTANT $zero, 0), (op $dst, $zero, $rhs)))>;
+ (pattern (op $dst, $zero, $rhs)))>;
def binop_left_to_zero: GICombineRule<
(defs root:$dst),
- (match (binop_left_to_zero_frags $dst, $zero, $rhs)),
+ (match (binop_left_to_zero_frags $dst, $zero, $rhs),
+ [{ return Helper.matchConstantOp(${zero}, 0); }]),
(apply (GIReplaceReg $dst, $zero))
>;
+def bitwise_binop_frags : GICombinePatFrag<
+ (outs root:$dst), (ins),
+ !foreach(op, [G_AND, G_OR, G_XOR],
+ (pattern (op $dst, $x, $y)))>;
+
// Fold `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`
def binop_with_neg : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
- (match (wip_match_opcode G_AND, G_OR, G_XOR):$root,
- [{ return Helper.matchBinopWithNeg(*${root}, ${matchinfo}); }]),
- (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])
+ (match (bitwise_binop_frags $root):$mi,
+ [{ return Helper.matchBinopWithNeg(*${mi}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])
>;
def urem_pow2_to_mask : GICombineRule<
@@ -812,25 +905,43 @@ def urem_pow2_to_mask : GICombineRule<
// Every binary operator that has constant folding. We currently do
// not have constant folding for G_FPOW, G_FMAXNUM_IEEE or
// G_FMINNUM_IEEE.
-def fold_binop_into_select : GICombineRule<
+// 26 opcodes exceed the 16-permutation cap for a single !foreach frag, so
+// splitted into two frags under a group keeping the original rule name.
+// All 26 share the (op $dst, $x, $y) shape, so the partition is arbitrary.
+def fold_binop_into_select_frags_a : GICombinePatFrag<
+ (outs root:$dst), (ins),
+ !foreach(op, [G_ADD, G_SUB, G_PTR_ADD, G_AND, G_OR, G_XOR,
+ G_SDIV, G_SREM, G_UDIV, G_UREM, G_LSHR, G_ASHR, G_SHL],
+ (pattern (op $dst, $x, $y)))>;
+def fold_binop_into_select_frags_b : GICombinePatFrag<
+ (outs root:$dst), (ins),
+ !foreach(op, [G_SMIN, G_SMAX, G_UMIN, G_UMAX,
+ G_FMUL, G_FADD, G_FSUB, G_FDIV, G_FREM,
+ G_FMINNUM, G_FMAXNUM, G_FMINIMUM, G_FMAXIMUM],
+ (pattern (op $dst, $x, $y)))>;
+
+class fold_binop_into_select_rule<GICombinePatFrag frags> : GICombineRule<
(defs root:$root, unsigned_matchinfo:$select_op_no),
- (match (wip_match_opcode
- G_ADD, G_SUB, G_PTR_ADD, G_AND, G_OR, G_XOR,
- G_SDIV, G_SREM, G_UDIV, G_UREM, G_LSHR, G_ASHR, G_SHL,
- G_SMIN, G_SMAX, G_UMIN, G_UMAX,
- G_FMUL, G_FADD, G_FSUB, G_FDIV, G_FREM,
- G_FMINNUM, G_FMAXNUM, G_FMINIMUM, G_FMAXIMUM):$root,
- [{ return Helper.matchFoldBinOpIntoSelect(*${root}, ${select_op_no}); }]),
- (apply [{ Helper.applyFoldBinOpIntoSelect(*${root}, ${select_op_no}); }])
->;
+ (match (frags $root):$mi,
+ [{ return Helper.matchFoldBinOpIntoSelect(*${mi}, ${select_op_no}); }]),
+ (apply [{ Helper.applyFoldBinOpIntoSelect(*${mi}, ${select_op_no}); }])>;
+
+def fold_binop_into_select_a : fold_binop_into_select_rule<fold_binop_into_select_frags_a>;
+def fold_binop_into_select_b : fold_binop_into_select_rule<fold_binop_into_select_frags_b>;
+def fold_binop_into_select : GICombineGroup<[fold_binop_into_select_a,
+ fold_binop_into_select_b]>;
// Transform d = [su]div(x, y) and r = [su]rem(x, y) - > d, r = [su]divrem(x, y)
def div_rem_to_divrem_matchdata : GIDefMatchData<"MachineInstr *">;
+def div_rem_to_divrem_frags : GICombinePatFrag<
+ (outs root:$dst), (ins),
+ !foreach(op, [G_SDIV, G_UDIV, G_SREM, G_UREM],
+ (pattern (op $dst, $x, $y)))>;
def div_rem_to_divrem : GICombineRule<
(defs root:$root, div_rem_to_divrem_matchdata:$matchinfo),
- (match (wip_match_opcode G_SDIV, G_UDIV, G_SREM, G_UREM):$root,
- [{ return Helper.matchCombineDivRem(*${root}, ${matchinfo}); }]),
- (apply [{ Helper.applyCombineDivRem(*${root}, ${matchinfo}); }])
+ (match (div_rem_to_divrem_frags $root):$mi,
+ [{ return Helper.matchCombineDivRem(*${mi}, ${matchinfo}); }]),
+ (apply [{ Helper.applyCombineDivRem(*${mi}, ${matchinfo}); }])
>;
// Fold (x op 0) - > 0
@@ -843,9 +954,9 @@ def binop_right_to_zero: GICombineRule<
// Erase stores of undef values.
def erase_undef_store : GICombineRule<
(defs root:$root),
- (match (wip_match_opcode G_STORE):$root,
- [{ return Helper.matchUndefStore(*${root}); }]),
- (apply [{ Helper.eraseInst(*${root}); }])
+ (match (G_IMPLICIT_DEF $val),
+ (G_STORE $val, $ptr):$root),
+ (apply (GIEraseRoot))
>;
// Fold ((0-A) + B) -> B - A, (A + (0-B)) -> A - B (negation is G_SUB 0, x).
@@ -921,14 +1032,30 @@ def p2i_to_i2p: GICombineRule<
(apply (GIReplaceReg $dst, $x))
>;
-// Fold ptr2int(int2ptr(x)) -> x
-def i2p_to_p2i: GICombineRule<
- (defs root:$dst, register_matchinfo:$info),
+// Fold ptr2int(int2ptr(x)) to x, extending or truncating when needed.
+def i2p_to_p2i_same_size: GICombineRule<
+ (defs root:$dst),
(match (G_INTTOPTR $t, $ptr),
(G_PTRTOINT $dst, $t):$mi,
- [{ ${info} = ${ptr}.getReg(); return true; }]),
- (apply [{ Helper.applyCombineP2IToI2P(*${mi}, ${info}); }])
->;
+ [{ return MRI.getType(${dst}.getReg()).getSizeInBits() ==
+ MRI.getType(${ptr}.getReg()).getSizeInBits(); }]),
+ (apply (COPY $dst, $ptr))>;
+def i2p_to_p2i_zext: GICombineRule<
+ (defs root:$dst),
+ (match (G_INTTOPTR $t, $ptr),
+ (G_PTRTOINT $dst, $t):$mi,
+ [{ return MRI.getType(${dst}.getReg()).getSizeInBits() >
+ MRI.getType(${ptr}.getReg()).getSizeInBits(); }]),
+ (apply (G_ZEXT $dst, $ptr))>;
+def i2p_to_p2i_trunc: GICombineRule<
+ (defs root:$dst),
+ (match (G_INTTOPTR $t, $ptr),
+ (G_PTRTOINT $dst, $t):$mi,
+ [{ return MRI.getType(${dst}.getReg()).getSizeInBits() <
+ MRI.getType(${ptr}.getReg()).getSizeInBits(); }]),
+ (apply (G_TRUNC $dst, $ptr))>;
+def i2p_to_p2i : GICombineGroup<[
+ i2p_to_p2i_same_size, i2p_to_p2i_zext, i2p_to_p2i_trunc]>;
// Fold add ptrtoint(x), y -> ptrtoint (ptr_add x), y
// !foreach covers both G_ADD operand orders; residue checks equal bitwidths.
@@ -967,9 +1094,9 @@ def const_ptradd_to_i2p: GICombineRule<
// Simplify: (logic_op (op x...), (op y...)) -> (op (logic_op x, y))
def hoist_logic_op_with_same_opcode_hands: GICombineRule <
(defs root:$root, instruction_steps_matchdata:$info),
- (match (wip_match_opcode G_AND, G_OR, G_XOR):$root,
- [{ return Helper.matchHoistLogicOpWithSameOpcodeHands(*${root}, ${info}); }]),
- (apply [{ Helper.applyBuildInstructionSteps(*${root}, ${info});}])
+ (match (bitwise_binop_frags $root):$mi,
+ [{ return Helper.matchHoistLogicOpWithSameOpcodeHands(*${mi}, ${info}); }]),
+ (apply [{ Helper.applyBuildInstructionSteps(*${mi}, ${info});}])
>;
// Fold ashr (shl x, C), C -> sext_inreg (C)
@@ -1000,7 +1127,7 @@ def neg_and_one_to_sext_inreg : GICombineRule<
// Fold and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0
def overlapping_and: GICombineRule <
(defs root:$root, build_fn_matchinfo:$info),
- (match (wip_match_opcode G_AND):$root,
+ (match (G_AND $dst, $x, $y):$root,
[{ return Helper.matchOverlappingAnd(*${root}, ${info}); }]),
(apply [{ Helper.applyBuildFn(*${root}, ${info}); }])
>;
@@ -1100,6 +1227,8 @@ def fneg_fneg_fold: GICombineRule <
(apply (GIReplaceReg $dst, $src))
>;
+// TODO: convert the G_UNMERGE_VALUES based-rules to structural MIR patterns once the
+// combiner emitter supports variadic-def roots (i.e. GIVariadic on defs).
// Fold (unmerge(merge x, y, z)) -> z, y, z.
def unmerge_merge : GICombineRule<
(defs root:$d, register_vector_matchinfo:$info),
@@ -1220,19 +1349,25 @@ def unmerge_zext_to_zext : GICombineRule<
/// Transform merge_x_undef -> anyext.
def merge_of_x_and_undef : GICombineRule <
- (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (defs root:$root),
(match (G_IMPLICIT_DEF $undef),
(G_MERGE_VALUES $root, $x, $undef):$MI,
- [{ return Helper.matchMergeXAndUndef(*${MI}, ${matchinfo}); }]),
- (apply [{ Helper.applyBuildFn(*${MI}, ${matchinfo}); }])>;
+ [{ return Helper.isLegalOrBeforeLegalizer(
+ {TargetOpcode::G_ANYEXT,
+ {MRI.getType(${root}.getReg()),
+ MRI.getType(${x}.getReg())}}); }]),
+ (apply (G_ANYEXT $root, $x))>;
/// Transform merge_x_zero -> zext.
def merge_of_x_and_zero : GICombineRule <
- (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (defs root:$root),
(match (G_CONSTANT $zero, 0),
(G_MERGE_VALUES $root, $x, $zero):$MI,
- [{ return Helper.matchMergeXAndZero(*${MI}, ${matchinfo}); }]),
- (apply [{ Helper.applyBuildFn(*${MI}, ${matchinfo}); }])>;
+ [{ return Helper.isLegalOrBeforeLegalizer(
+ {TargetOpcode::G_ZEXT,
+ {MRI.getType(${root}.getReg()),
+ MRI.getType(${x}.getReg())}}); }]),
+ (apply (G_ZEXT $root, $x))>;
// Transform build_vector(unmerge(src, 0), ... unmerge(src, n), undef, ..., undef)
// => concat_vectors(src, undef)
@@ -1284,12 +1419,27 @@ def xor_of_and_with_same_reg: GICombineRule <
(apply [{ Helper.applyXorOfAndWithSameReg(*${root}, ${matchinfo}); }])
>;
-// Transform (ptr_add 0, x) -> (int_to_ptr x)
-def ptr_add_with_zero: GICombineRule<
+// Transform scalar (ptr_add 0, x) -> (int_to_ptr x).
+def ptr_add_with_zero_scalar: GICombineRule<
+ (defs root:$root),
+ (match (G_PTR_ADD $dst, 0, $offset):$root,
+ [{
+ LLT Ty = MRI.getType(${dst}.getReg());
+ return Ty.isPointer() &&
+ !${root}->getMF()->getDataLayout().isNonIntegralAddressSpace(
+ Ty.getAddressSpace());
+ }]),
+ (apply (G_INTTOPTR $dst, $offset))>;
+
+// A zero vector may be represented by a G_BUILD_VECTOR splat, which cannot be
+// expressed as a fixed-shape pattern.
+def ptr_add_with_zero_vector: GICombineRule<
(defs root:$root),
(match (G_PTR_ADD $dst, $base, $offset):$root,
- [{ return Helper.matchPtrAddZero(*${root}); }]),
+ [{ return Helper.matchPtrAddZeroVector(*${root}); }]),
(apply (G_INTTOPTR $dst, $offset))>;
+def ptr_add_with_zero : GICombineGroup<[
+ ptr_add_with_zero_scalar, ptr_add_with_zero_vector]>;
def combine_insert_vec_elts_build_vector : GICombineRule<
(defs root:$root, register_vector_matchinfo:$info),
@@ -1352,12 +1502,22 @@ def funnel_shift_op_frags : GICombinePatFrag<
(outs root:$dst), (ins),
!foreach(op, [G_FSHL, G_FSHR], (pattern (op $dst, $x, $y, $amt)))>;
-def funnel_shift_to_rotate : GICombineRule<
+class funnel_shift_to_rotate_rule<Instruction fsh, Instruction rot>
+ : GICombineRule<
(defs root:$dst),
- (match (funnel_shift_op_frags $dst):$root,
- [{ return Helper.matchFunnelShiftToRotate(*${root}); }]),
- (apply [{ Helper.applyFunnelShiftToRotate(*${root}); }])
->;
+ (match (fsh $dst, $x, $x, $amt):$root,
+ [{ return Helper.isLegalOrBeforeLegalizer(
+ {TargetOpcode::}] # rot # [{,
+ {MRI.getType(${x}.getReg()),
+ MRI.getType(${x}.getReg())}}); }]),
+ (apply (rot $dst, $x, $amt))>;
+def funnel_shift_to_rotate_left
+ : funnel_shift_to_rotate_rule<G_FSHL, G_ROTL>;
+def funnel_shift_to_rotate_right
+ : funnel_shift_to_rotate_rule<G_FSHR, G_ROTR>;
+
+def funnel_shift_to_rotate : GICombineGroup<[
+ funnel_shift_to_rotate_left, funnel_shift_to_rotate_right]>;
// Fold fshr x, y, 0 -> y
def funnel_shift_right_zero: GICombineRule<
@@ -1637,13 +1797,23 @@ def reassoc_comm_binops : GICombineRule<
def reassocs : GICombineGroup<[reassoc_ptradd, reassoc_comm_binops]>;
// Constant fold operations.
+def constant_fold_binop_frags : GICombinePatFrag<
+ (outs root:$dst), (ins),
+ !foreach(op,
+ [G_ADD, G_PTR_ADD, G_AND, G_ASHR, G_LSHR, G_MUL, G_OR,
+ G_SHL, G_SUB, G_XOR, G_UDIV, G_SDIV, G_UREM, G_SREM,
+ G_SMIN, G_SMAX, G_UMIN, G_UMAX],
+ (pattern (op $dst, $src0, $src1)))>;
+
def constant_fold_binop : GICombineRule<
(defs root:$d, apint_matchinfo:$matchinfo),
- (match (wip_match_opcode G_ADD, G_PTR_ADD, G_AND, G_ASHR, G_LSHR, G_MUL, G_OR,
- G_SHL, G_SUB, G_XOR, G_UDIV, G_SDIV, G_UREM, G_SREM,
- G_SMIN, G_SMAX, G_UMIN, G_UMAX):$d,
- [{ return Helper.matchConstantFoldBinOp(*${d}, ${matchinfo}); }]),
- (apply [{ Helper.replaceInstWithConstant(*${d}, ${matchinfo}); }])>;
+ (match (constant_fold_binop_frags $d):$mi,
+ [{ return Helper.matchConstantFoldBinOp(*${mi}, ${matchinfo}); }]),
+ (apply [{ Helper.replaceInstWithConstant(*${mi}, ${matchinfo}); }])> {
+ // 18 opcode alternatives exceed the default 16-permutation cap, so lift it.
+ // A single frag (one rule id) keeps the rule name valid for --disable-rule.
+ let MaxPermutations = -1;
+}
def constant_fold_fp_binop_frags : GICombinePatFrag<
(outs root:$dst), (ins),
@@ -1693,27 +1863,43 @@ def constant_fold_unary_int_op : GICombineRule<
[{ return Helper.matchConstantFoldUnaryIntOp(*${mi}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])>;
-def mulo_by_2: GICombineRule<
- (defs root:$root, build_fn_matchinfo:$matchinfo),
- (match (wip_match_opcode G_UMULO, G_SMULO):$root,
- [{ return Helper.matchMulOBy2(*${root}, ${matchinfo}); }]),
- (apply [{ Helper.applyBuildFnNoErase(*${root}, ${matchinfo}); }])>;
-
-def mulo_by_0: GICombineRule<
+// G_UMULO/G_SMULO are two-def (result + carry) roots. A GICombinePatFrag root
+// must have a single root def, so instead each opcode is a per-opcode rule
+// (matching both defs structurally, like adde_to_addo) gathered under a group
+// keeping the original rule name.
+class mulo_by_2_rule<Instruction op, Instruction addo> : GICombineRule<
+ (defs root:$root),
+ (match (op $dst, $carry, $x, $rhs):$root,
+ [{ return Helper.matchConstantOp(${rhs}, 2); }]),
+ (apply (addo $dst, $carry, $x, $x, (MIFlags $root)))>;
+def mulo_by_2_umulo : mulo_by_2_rule<G_UMULO, G_UADDO>;
+def mulo_by_2_smulo : mulo_by_2_rule<G_SMULO, G_SADDO>;
+def mulo_by_2 : GICombineGroup<[mulo_by_2_umulo, mulo_by_2_smulo]>;
+
+class mulo_by_0_rule<Instruction op> : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
- (match (wip_match_opcode G_UMULO, G_SMULO):$root,
+ (match (op $dst, $carry, $x, $y):$root,
[{ return Helper.matchMulOBy0(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+def mulo_by_0_umulo : mulo_by_0_rule<G_UMULO>;
+def mulo_by_0_smulo : mulo_by_0_rule<G_SMULO>;
+def mulo_by_0 : GICombineGroup<[mulo_by_0_umulo, mulo_by_0_smulo]>;
// Transform (uadde x, y, 0) -> (uaddo x, y)
// (sadde x, y, 0) -> (saddo x, y)
// (usube x, y, 0) -> (usubo x, y)
// (ssube x, y, 0) -> (ssubo x, y)
-def adde_to_addo: GICombineRule<
- (defs root:$root, build_fn_matchinfo:$matchinfo),
- (match (wip_match_opcode G_UADDE, G_SADDE, G_USUBE, G_SSUBE):$root,
- [{ return Helper.matchAddEToAddO(*${root}, ${matchinfo}); }]),
- (apply [{ Helper.applyBuildFnNoErase(*${root}, ${matchinfo}); }])>;
+class adde_to_addo_rule<Instruction inop, Instruction outop> : GICombineRule<
+ (defs root:$root),
+ (match (inop $dst, $carry_out, $x, $y, $carry_in):$root,
+ [{ return Helper.matchConstantOp(${carry_in}, 0); }]),
+ (apply (outop $dst, $carry_out, $x, $y, (MIFlags $root)))>;
+def adde_to_addo_uadde : adde_to_addo_rule<G_UADDE, G_UADDO>;
+def adde_to_addo_sadde : adde_to_addo_rule<G_SADDE, G_SADDO>;
+def adde_to_addo_usube : adde_to_addo_rule<G_USUBE, G_USUBO>;
+def adde_to_addo_ssube : adde_to_addo_rule<G_SSUBE, G_SSUBO>;
+def adde_to_addo: GICombineGroup<[adde_to_addo_uadde, adde_to_addo_sadde,
+ adde_to_addo_usube, adde_to_addo_ssube]>;
def mulh_to_lshr : GICombineRule<
(defs root:$root),
@@ -1752,14 +1938,32 @@ def truncusatu_to_fptouisat : GICombineRule<
def truncsat_combines : GICombineGroup<[trunc_ssats, trunc_ssatu, trunc_usatu, truncusatu_to_fptouisat]>;
-// fold (fadd x, fneg(y)) -> (fsub x, y)
-// fold (fsub x, fneg(y)) -> (fadd x, y)
-// These need a legality check on the swapped opcode, so stay in C++.
-def redundant_neg_operands_fadd_fsub: GICombineRule<
- (defs root:$root, build_fn_matchinfo:$matchinfo),
- (match (wip_match_opcode G_FADD, G_FSUB):$root,
- [{ return Helper.matchRedundantNegOperands(*${root}, ${matchinfo}); }]),
- (apply [{ Helper.applyBuildFnNoErase(*${root}, ${matchinfo}); }])>;
+// fold (fadd x, fneg(y)) -> (fsub x, y) (G_FADD is commutative, so both
+// operand orders are covered by two rules). The swapped-opcode legality
+// check is inline; flags are carried via MIFlags.
+def redundant_neg_operands_fadd_rhs: GICombineRule<
+ (defs root:$dst),
+ (match (G_FNEG $negy, $y),
+ (G_FADD $dst, $x, $negy):$mi,
+ [{ return Helper.isLegalOrBeforeLegalizer({TargetOpcode::G_FSUB,
+ {MRI.getType(${dst}.getReg())}}); }]),
+ (apply (G_FSUB $dst, $x, $y, (MIFlags $mi)))>;
+def redundant_neg_operands_fadd_lhs: GICombineRule<
+ (defs root:$dst),
+ (match (G_FNEG $negy, $y),
+ (G_FADD $dst, $negy, $x):$mi,
+ [{ return Helper.isLegalOrBeforeLegalizer({TargetOpcode::G_FSUB,
+ {MRI.getType(${dst}.getReg())}}); }]),
+ (apply (G_FSUB $dst, $x, $y, (MIFlags $mi)))>;
+
+// fold (fsub x, fneg(y)) -> (fadd x, y) (G_FSUB is not commutative).
+def redundant_neg_operands_fsub: GICombineRule<
+ (defs root:$dst),
+ (match (G_FNEG $negy, $y),
+ (G_FSUB $dst, $x, $negy):$mi,
+ [{ return Helper.isLegalOrBeforeLegalizer({TargetOpcode::G_FADD,
+ {MRI.getType(${dst}.getReg())}}); }]),
+ (apply (G_FADD $dst, $x, $y, (MIFlags $mi)))>;
// fold (fmul (fneg x), (fneg y)) -> (fmul x, y)
// fold (fdiv (fneg x), (fneg y)) -> (fdiv x, y)
@@ -1782,15 +1986,18 @@ foreach fmaop = [G_FMAD, G_FMA] in {
}
def redundant_neg_operands : GICombineGroup<
- [redundant_neg_operands_fadd_fsub, G_FMUL_fneg_fneg, G_FDIV_fneg_fneg,
+ [redundant_neg_operands_fadd_rhs, redundant_neg_operands_fadd_lhs,
+ redundant_neg_operands_fsub,
+ G_FMUL_fneg_fneg, G_FDIV_fneg_fneg,
G_FMAD_fneg_fneg, G_FMA_fneg_fneg]>;
// Transform (fsub +-0.0, X) -> (fneg X)
def fsub_to_fneg: GICombineRule<
- (defs root:$root, register_matchinfo:$matchinfo),
- (match (G_FSUB $dst, $src1, $src2):$root,
- [{ return Helper.matchFsubToFneg(*${root}, ${matchinfo}); }]),
- (apply [{ Helper.applyFsubToFneg(*${root}, ${matchinfo}); }])>;
+ (defs root:$dst),
+ (match (G_FSUB $dst, $lhs, $rhs):$root,
+ [{ return Helper.matchFsubToFneg(*${root}); }]),
+ (apply (G_FCANONICALIZE GITypeOf<"$dst">:$canon, $rhs),
+ (G_FNEG $dst, $canon))>;
// Transform (fadd x, (fmul y, z)) -> (fma y, z, x)
// (fadd x, (fmul y, z)) -> (fmad y, z, x)
@@ -1870,16 +2077,68 @@ def combine_fsub_fpext_fneg_fmul_to_fmad_or_fma: GICombineRule<
*${root}, ${info}); }]),
(apply [{ Helper.applyBuildFn(*${root}, ${info}); }])>;
-def combine_minmax_nan_frags : GICombinePatFrag<
- (outs root:$dst), (ins),
- !foreach(op, [G_FMINNUM, G_FMAXNUM, G_FMINIMUM, G_FMAXIMUM],
- (pattern (op $dst, $src0, $src1)))>;
+// Fold minnum/maxnum/minimum/maximum with a NaN operand.
+// G_FMINNUM/G_FMAXNUM discard NaN (keep the other operand).
+// G_FMINIMUM/G_FMAXIMUM return the NaN (IEEE 754-2019).
+// Opcode fan is a patfrag; $lhs/$rhs are ins (uses), not outs (L4).
+// Split by family × which operand is NaN (L7). When both operands are NaN,
+// only the LHS rule fires so the two sides do not compete.
+class combine_minmax_nan_frags<list<Instruction> ops> : GICombinePatFrag<
+ (outs root:$dst), (ins $lhs, $rhs),
+ !foreach(op, ops, (pattern (op $dst, $lhs, $rhs)))>;
+def minmax_nan_num_frags
+ : combine_minmax_nan_frags<[G_FMINNUM, G_FMAXNUM]>;
+def minmax_nan_ieee_frags
+ : combine_minmax_nan_frags<[G_FMINIMUM, G_FMAXIMUM]>;
+
+class minmax_nan_lhs_keep_rhs<GICombinePatFrag frags> : GICombineRule<
+ (defs root:$dst),
+ (match (frags $dst, $lhs, $rhs),
+ [{
+ const ConstantFP *C = getConstantFPVRegVal(${lhs}.getReg(), MRI);
+ return C && C->getValueAPF().isNaN();
+ }]),
+ (apply (GIReplaceReg $dst, $rhs))>;
+
+class minmax_nan_rhs_keep_lhs<GICombinePatFrag frags> : GICombineRule<
+ (defs root:$dst),
+ (match (frags $dst, $lhs, $rhs),
+ [{
+ const ConstantFP *RHS = getConstantFPVRegVal(${rhs}.getReg(), MRI);
+ const ConstantFP *LHS = getConstantFPVRegVal(${lhs}.getReg(), MRI);
+ return RHS && RHS->getValueAPF().isNaN() &&
+ (!LHS || !LHS->getValueAPF().isNaN());
+ }]),
+ (apply (GIReplaceReg $dst, $lhs))>;
+
+class minmax_nan_lhs_keep_lhs<GICombinePatFrag frags> : GICombineRule<
+ (defs root:$dst),
+ (match (frags $dst, $lhs, $rhs),
+ [{
+ const ConstantFP *C = getConstantFPVRegVal(${lhs}.getReg(), MRI);
+ return C && C->getValueAPF().isNaN();
+ }]),
+ (apply (GIReplaceReg $dst, $lhs))>;
-def combine_minmax_nan: GICombineRule<
- (defs root:$dst, unsigned_matchinfo:$info),
- (match (combine_minmax_nan_frags $dst):$root,
- [{ return Helper.matchCombineFMinMaxNaN(*${root}, ${info}); }]),
- (apply [{ Helper.replaceSingleDefInstWithOperand(*${root}, ${info}); }])>;
+class minmax_nan_rhs_keep_rhs<GICombinePatFrag frags> : GICombineRule<
+ (defs root:$dst),
+ (match (frags $dst, $lhs, $rhs),
+ [{
+ const ConstantFP *RHS = getConstantFPVRegVal(${rhs}.getReg(), MRI);
+ const ConstantFP *LHS = getConstantFPVRegVal(${lhs}.getReg(), MRI);
+ return RHS && RHS->getValueAPF().isNaN() &&
+ (!LHS || !LHS->getValueAPF().isNaN());
+ }]),
+ (apply (GIReplaceReg $dst, $rhs))>;
+
+def minmax_nan_num_lhs : minmax_nan_lhs_keep_rhs<minmax_nan_num_frags>;
+def minmax_nan_num_rhs : minmax_nan_rhs_keep_lhs<minmax_nan_num_frags>;
+def minmax_nan_ieee_lhs : minmax_nan_lhs_keep_lhs<minmax_nan_ieee_frags>;
+def minmax_nan_ieee_rhs : minmax_nan_rhs_keep_rhs<minmax_nan_ieee_frags>;
+
+def combine_minmax_nan: GICombineGroup<[
+ minmax_nan_num_lhs, minmax_nan_num_rhs,
+ minmax_nan_ieee_lhs, minmax_nan_ieee_rhs]>;
// Combine multiple FDIVs with the same divisor into multiple FMULs by the
// reciprocal.
@@ -1902,11 +2161,15 @@ def add_sub_reg: GICombineRule <
(match (add_sub_reg_frags $dst, $src)),
(apply (GIReplaceReg $dst, $src))>;
-def buildvector_identity_fold : GICombineRule<
+class buildvector_identity_fold_rule<Instruction op> : GICombineRule<
(defs root:$build_vector, register_matchinfo:$matchinfo),
- (match (wip_match_opcode G_BUILD_VECTOR_TRUNC, G_BUILD_VECTOR):$build_vector,
- [{ return Helper.matchBuildVectorIdentityFold(*${build_vector}, ${matchinfo}); }]),
- (apply [{ Helper.replaceSingleDefInstWithReg(*${build_vector}, ${matchinfo}); }])>;
+ (match (op $build_vector, GIVariadic<>:$srcs):$mi,
+ [{ return Helper.matchBuildVectorIdentityFold(*${mi}, ${matchinfo}); }]),
+ (apply [{ Helper.replaceSingleDefInstWithReg(*${mi}, ${matchinfo}); }])>;
+def buildvector_identity_fold_bv : buildvector_identity_fold_rule<G_BUILD_VECTOR>;
+def buildvector_identity_fold_bvt : buildvector_identity_fold_rule<G_BUILD_VECTOR_TRUNC>;
+def buildvector_identity_fold : GICombineGroup<[buildvector_identity_fold_bv,
+ buildvector_identity_fold_bvt]>;
def trunc_buildvector_fold : GICombineRule<
(defs root:$op, register_matchinfo:$matchinfo),
@@ -1925,17 +2188,32 @@ def trunc_lshr_buildvector_fold : GICombineRule<
// (x + y) - x -> y
// x - (y + x) -> 0 - y
// x - (x + z) -> 0 - z
-def sub_add_reg: GICombineRule <
+def sub_add_reg_rhs: GICombineRule<
+ (defs root:$dst),
+ (match (G_ADD $sum, $x, $y),
+ (G_SUB $dst, $sum, $y)),
+ (apply (GIReplaceReg $dst, $x))>;
+
+def sub_add_reg_lhs: GICombineRule<
+ (defs root:$dst),
+ (match (G_ADD $sum, $x, $y),
+ (G_SUB $dst, $sum, $x)),
+ (apply (GIReplaceReg $dst, $y))>;
+
+// Keep the value-equivalent and negating cases in C++.
+def sub_add_reg_fallback: GICombineRule <
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_SUB $dst, $src1, $src2):$root,
[{ return Helper.matchSubAddSameReg(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+def sub_add_reg : GICombineGroup<[
+ sub_add_reg_rhs, sub_add_reg_lhs, sub_add_reg_fallback]>;
def bitcast_bitcast_fold : GICombineRule<
(defs root:$dst),
(match (G_BITCAST $dst, $src1):$op, (G_BITCAST $src1, $src0),
[{ return MRI.getType(${src0}.getReg()) == MRI.getType(${dst}.getReg()); }]),
- (apply [{ Helper.replaceSingleDefInstWithReg(*${op}, ${src0}.getReg()); }])>;
+ (apply (GIReplaceReg $dst, $src0))>;
def bitcast_bitcast_to_bitcast : GICombineRule<
(defs root:$dst),
@@ -1947,7 +2225,7 @@ def fptrunc_fpext_fold : GICombineRule<
(defs root:$dst),
(match (G_FPTRUNC $dst, $src1):$op, (G_FPEXT $src1, $src0),
[{ return MRI.getType(${src0}.getReg()) == MRI.getType(${dst}.getReg()); }]),
- (apply [{ Helper.replaceSingleDefInstWithReg(*${op}, ${src0}.getReg()); }])>;
+ (apply (GIReplaceReg $dst, $src0))>;
def select_to_minmax: GICombineRule<
@@ -1987,17 +2265,38 @@ def match_ors : GICombineRule<
[{ return Helper.matchOr(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
-def match_addos : GICombineRule<
+class match_addos_dead_carry_rule<Instruction op> : GICombineRule<
+ (defs root:$root),
+ (match (op $dst, $carry, $x, $y):$root,
+ [{ return MRI.use_nodbg_empty(${carry}.getReg()) &&
+ Helper.isLegalOrBeforeLegalizer(
+ {TargetOpcode::G_ADD,
+ {MRI.getType(${dst}.getReg())}}); }]),
+ (apply (G_ADD $dst, $x, $y),
+ (G_IMPLICIT_DEF $carry))>;
+
+class match_addos_rule<Instruction op> : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
- (match (wip_match_opcode G_SADDO, G_UADDO):$root,
+ (match (op $dst, $carry, $x, $y):$root,
[{ return Helper.matchAddOverflow(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
-
-def match_subo_no_overflow : GICombineRule<
+def match_addos_saddo_dead_carry : match_addos_dead_carry_rule<G_SADDO>;
+def match_addos_uaddo_dead_carry : match_addos_dead_carry_rule<G_UADDO>;
+def match_addos_saddo : match_addos_rule<G_SADDO>;
+def match_addos_uaddo : match_addos_rule<G_UADDO>;
+def match_addos : GICombineGroup<[
+ match_addos_saddo_dead_carry, match_addos_uaddo_dead_carry,
+ match_addos_saddo, match_addos_uaddo]>;
+
+class match_subo_no_overflow_rule<Instruction op> : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
- (match (wip_match_opcode G_SSUBO, G_USUBO):$root,
+ (match (op $dst, $carry, $x, $y):$root,
[{ return Helper.matchSuboCarryOut(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+def match_subo_no_overflow_ssubo : match_subo_no_overflow_rule<G_SSUBO>;
+def match_subo_no_overflow_usubo : match_subo_no_overflow_rule<G_USUBO>;
+def match_subo_no_overflow : GICombineGroup<[match_subo_no_overflow_ssubo,
+ match_subo_no_overflow_usubo]>;
// extractelement(undef_vector, idx) and extractelement(vector, undef_idx) are
// both undef. Either undef operand is matched by the shared frag.
@@ -2009,10 +2308,10 @@ def extract_vector_element_undef : GICombineRule <
>;
def match_extract_of_element : GICombineRule<
- (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (defs root:$root),
(match (G_EXTRACT_VECTOR_ELT $dst, $src, $idx):$root,
- [{ return Helper.matchExtractVectorElement(*${root}, ${matchinfo}); }]),
- (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+ [{ return Helper.matchExtractVectorElement(*${root}); }]),
+ (apply (G_IMPLICIT_DEF $dst))>;
def extract_vector_element_not_const : GICombineRule<
(defs root:$root),
@@ -2173,10 +2472,10 @@ def insert_vector_element_extract_vector_element : GICombineRule<
(apply (GIReplaceReg $root, $src))>;
def insert_vector_elt_oob : GICombineRule<
- (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (defs root:$root),
(match (G_INSERT_VECTOR_ELT $dst, $src, $elt, $idx):$root,
- [{ return Helper.matchInsertVectorElementOOB(*${root}, ${matchinfo}); }]),
- (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+ [{ return Helper.matchInsertVectorElementOOB(*${root}); }]),
+ (apply (G_IMPLICIT_DEF $dst))>;
// Combine v8i8 (buildvector i8 (trunc(unmerge)), i8 (trunc), i8 (trunc), i8 (trunc), undef, undef, undef, undef)
def combine_use_vector_truncate : GICombineRule<
@@ -2608,14 +2907,17 @@ def ctls_combines : GICombineGroup<[
// Switch G_CTLZ / G_CTTZ to *_ZERO_POISON when the input is provably nonzero
// and the target supports the variant. The zero-poison variants compute the
// same result for nonzero inputs and can let targets skip zero-input fixups.
-class count_zero_to_zero_poison<Instruction opc> : GICombineRule<
+class count_zero_to_zero_poison<Instruction opc, Instruction zero_poison_opc>
+ : GICombineRule<
(defs root:$root),
(match (opc $dst, $src):$root,
[{ return Helper.matchCountZeroToZeroPoison(*${root}); }]),
- (apply [{ Helper.applyCountZeroToZeroPoison(*${root}); }])>;
+ (apply (zero_poison_opc $dst, $src, (MIFlags $root)))>;
-def ctlz_to_ctlz_zero_poison : count_zero_to_zero_poison<G_CTLZ>;
-def cttz_to_cttz_zero_poison : count_zero_to_zero_poison<G_CTTZ>;
+def ctlz_to_ctlz_zero_poison
+ : count_zero_to_zero_poison<G_CTLZ, G_CTLZ_ZERO_POISON>;
+def cttz_to_cttz_zero_poison
+ : count_zero_to_zero_poison<G_CTTZ, G_CTTZ_ZERO_POISON>;
def count_zero_poison_combines : GICombineGroup<[
ctlz_to_ctlz_zero_poison, cttz_to_cttz_zero_poison
@@ -2702,7 +3004,10 @@ def avgfloor_u_match : GICombineRule<
(G_ZEXT $ext2, $y),
(G_ADD $sum, $ext1, $ext2),
(G_LSHR $dst, $sum, 1):$mi,
- [{ return Helper.matchAVG(*${mi}, MRI, ${x}.getReg(), ${y}.getReg(), TargetOpcode::G_UAVGFLOOR); }]),
+ [{ return MRI.getType(${x}.getReg()) ==
+ MRI.getType(${y}.getReg()) &&
+ Helper.isLegal({TargetOpcode::G_UAVGFLOOR,
+ {MRI.getType(${x}.getReg())}}); }]),
(apply (G_UAVGFLOOR $tmp, $x, $y),
(G_ZEXT $dst, $tmp))
>;
@@ -2713,7 +3018,10 @@ def avgfloor_s_match : GICombineRule<
(G_SEXT $ext2, $y),
(G_ADD $sum, $ext1, $ext2),
(G_ASHR $dst, $sum, 1):$mi,
- [{ return Helper.matchAVG(*${mi}, MRI, ${x}.getReg(), ${y}.getReg(), TargetOpcode::G_SAVGFLOOR); }]),
+ [{ return MRI.getType(${x}.getReg()) ==
+ MRI.getType(${y}.getReg()) &&
+ Helper.isLegal({TargetOpcode::G_SAVGFLOOR,
+ {MRI.getType(${x}.getReg())}}); }]),
(apply (G_SAVGFLOOR $tmp, $x, $y),
(G_SEXT $dst, $tmp))
>;
@@ -2725,7 +3033,10 @@ def avgceil_u_match : GICombineRule<
(G_ADD $inner, $ext1, $ext2),
(G_ADD $sum, $inner, 1),
(G_LSHR $dst, $sum, 1):$mi,
- [{ return Helper.matchAVG(*${mi}, MRI, ${x}.getReg(), ${y}.getReg(), TargetOpcode::G_UAVGCEIL); }]),
+ [{ return MRI.getType(${x}.getReg()) ==
+ MRI.getType(${y}.getReg()) &&
+ Helper.isLegal({TargetOpcode::G_UAVGCEIL,
+ {MRI.getType(${x}.getReg())}}); }]),
(apply (G_UAVGCEIL $tmp, $x, $y),
(G_ZEXT $dst, $tmp))
>;
@@ -2737,7 +3048,10 @@ def avgceil_s_match : GICombineRule<
(G_ADD $inner, $ext1, $ext2),
(G_ADD $sum, $inner, 1),
(G_ASHR $dst, $sum, 1):$mi,
- [{ return Helper.matchAVG(*${mi}, MRI, ${x}.getReg(), ${y}.getReg(), TargetOpcode::G_SAVGCEIL); }]),
+ [{ return MRI.getType(${x}.getReg()) ==
+ MRI.getType(${y}.getReg()) &&
+ Helper.isLegal({TargetOpcode::G_SAVGCEIL,
+ {MRI.getType(${x}.getReg())}}); }]),
(apply (G_SAVGCEIL $tmp, $x, $y),
(G_SEXT $dst, $tmp))
>;
diff --git a/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt b/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt
index a9e29b9182bff..4a9be26da963a 100644
--- a/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt
+++ b/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt
@@ -6,7 +6,6 @@ add_llvm_component_library(LLVMGlobalISel
GlobalISel.cpp
Combiner.cpp
CombinerHelper.cpp
- CombinerHelperArtifacts.cpp
CombinerHelperCasts.cpp
CombinerHelperCompares.cpp
CombinerHelperVectorOps.cpp
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index cd672ed0b6f53..20984dd13af1d 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -2297,25 +2297,6 @@ void CombinerHelper::applyCombineShlOfExtend(
MI.eraseFromParent();
}
-bool CombinerHelper::matchCombineMergeUnmerge(MachineInstr &MI,
- Register &MatchInfo) const {
- GMerge &Merge = cast<GMerge>(MI);
- SmallVector<Register, 16> MergedValues;
- for (unsigned I = 0; I < Merge.getNumSources(); ++I)
- MergedValues.emplace_back(Merge.getSourceReg(I));
-
- auto *Unmerge = getOpcodeDef<GUnmerge>(MergedValues[0], MRI);
- if (!Unmerge || Unmerge->getNumDefs() != Merge.getNumSources())
- return false;
-
- for (unsigned I = 0; I < MergedValues.size(); ++I)
- if (MergedValues[I] != Unmerge->getReg(I))
- return false;
-
- MatchInfo = Unmerge->getSourceReg();
- return true;
-}
-
static Register peekThroughBitcast(Register Reg,
const MachineRegisterInfo &MRI) {
while (mi_match(Reg, MRI, m_GBitcast(m_Reg(Reg))))
@@ -2615,87 +2596,6 @@ bool CombinerHelper::tryCombineShiftToUnmerge(
return false;
}
-void CombinerHelper::applyCombineP2IToI2P(MachineInstr &MI,
- Register &Reg) const {
- assert(MI.getOpcode() == TargetOpcode::G_PTRTOINT && "Expected a G_PTRTOINT");
- Register DstReg = MI.getOperand(0).getReg();
- Builder.buildZExtOrTrunc(DstReg, Reg);
- MI.eraseFromParent();
-}
-
-bool CombinerHelper::matchCombineAddP2IToPtrAdd(
- MachineInstr &MI, std::pair<Register, bool> &PtrReg) const {
- assert(MI.getOpcode() == TargetOpcode::G_ADD);
- Register LHS = MI.getOperand(1).getReg();
- Register RHS = MI.getOperand(2).getReg();
- LLT IntTy = MRI.getType(LHS);
-
- // G_PTR_ADD always has the pointer in the LHS, so we may need to commute the
- // instruction.
- PtrReg.second = false;
- for (Register SrcReg : {LHS, RHS}) {
- if (mi_match(SrcReg, MRI, m_GPtrToInt(m_Reg(PtrReg.first)))) {
- // Don't handle cases where the integer is implicitly converted to the
- // pointer width.
- LLT PtrTy = MRI.getType(PtrReg.first);
- if (PtrTy.getScalarSizeInBits() == IntTy.getScalarSizeInBits())
- return true;
- }
-
- PtrReg.second = true;
- }
-
- return false;
-}
-
-void CombinerHelper::applyCombineAddP2IToPtrAdd(
- MachineInstr &MI, std::pair<Register, bool> &PtrReg) const {
- Register Dst = MI.getOperand(0).getReg();
- Register LHS = MI.getOperand(1).getReg();
- Register RHS = MI.getOperand(2).getReg();
-
- const bool DoCommute = PtrReg.second;
- if (DoCommute)
- std::swap(LHS, RHS);
- LHS = PtrReg.first;
-
- LLT PtrTy = MRI.getType(LHS);
-
- auto PtrAdd = Builder.buildPtrAdd(PtrTy, LHS, RHS);
- Builder.buildPtrToInt(Dst, PtrAdd);
- MI.eraseFromParent();
-}
-
-bool CombinerHelper::matchCombineConstPtrAddToI2P(MachineInstr &MI,
- APInt &NewCst) const {
- auto &PtrAdd = cast<GPtrAdd>(MI);
- Register LHS = PtrAdd.getBaseReg();
- Register RHS = PtrAdd.getOffsetReg();
- MachineRegisterInfo &MRI = Builder.getMF().getRegInfo();
-
- if (auto RHSCst = getIConstantVRegVal(RHS, MRI)) {
- APInt Cst;
- if (mi_match(LHS, MRI, m_GIntToPtr(m_ICst(Cst)))) {
- auto DstTy = MRI.getType(PtrAdd.getReg(0));
- // G_INTTOPTR uses zero-extension
- NewCst = Cst.zextOrTrunc(DstTy.getSizeInBits());
- NewCst += RHSCst->sextOrTrunc(DstTy.getSizeInBits());
- return true;
- }
- }
-
- return false;
-}
-
-void CombinerHelper::applyCombineConstPtrAddToI2P(MachineInstr &MI,
- APInt &NewCst) const {
- auto &PtrAdd = cast<GPtrAdd>(MI);
- Register Dst = PtrAdd.getReg(0);
-
- Builder.buildConstant(Dst, NewCst);
- PtrAdd.eraseFromParent();
-}
-
bool CombinerHelper::matchCombineAnyExtTrunc(MachineInstr &MI,
Register &Reg) const {
assert(MI.getOpcode() == TargetOpcode::G_ANYEXT && "Expected a G_ANYEXT");
@@ -2847,18 +2747,6 @@ bool CombinerHelper::matchUndefShuffleVectorMask(MachineInstr &MI) const {
return all_of(Mask, [](int Elt) { return Elt < 0; });
}
-bool CombinerHelper::matchUndefStore(MachineInstr &MI) const {
- assert(MI.getOpcode() == TargetOpcode::G_STORE);
- return getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MI.getOperand(0).getReg(),
- MRI);
-}
-
-bool CombinerHelper::matchUndefSelectCmp(MachineInstr &MI) const {
- assert(MI.getOpcode() == TargetOpcode::G_SELECT);
- return getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MI.getOperand(1).getReg(),
- MRI);
-}
-
bool CombinerHelper::matchInsertExtractVecEltOutOfBounds(
MachineInstr &MI) const {
assert((MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT ||
@@ -2876,16 +2764,6 @@ bool CombinerHelper::matchInsertExtractVecEltOutOfBounds(
return Idx->getZExtValue() >= VecTy.getNumElements();
}
-bool CombinerHelper::matchConstantSelectCmp(MachineInstr &MI,
- unsigned &OpIdx) const {
- GSelect &SelMI = cast<GSelect>(MI);
- auto Cst = isConstantOrConstantSplatVector(SelMI.getCondReg(), MRI);
- if (!Cst)
- return false;
- OpIdx = Cst->isZero() ? 3 : 2;
- return true;
-}
-
void CombinerHelper::eraseInst(MachineInstr &MI) const { MI.eraseFromParent(); }
bool CombinerHelper::matchEqualDefs(const MachineOperand &MOP1,
@@ -3108,26 +2986,6 @@ void CombinerHelper::replaceInstWithUndef(MachineInstr &MI) const {
MI.eraseFromParent();
}
-bool CombinerHelper::matchSimplifyAddToSub(
- MachineInstr &MI, std::tuple<Register, Register> &MatchInfo) const {
- Register LHS = MI.getOperand(1).getReg();
- Register RHS = MI.getOperand(2).getReg();
- Register &NewLHS = std::get<0>(MatchInfo);
- Register &NewRHS = std::get<1>(MatchInfo);
-
- // Helper lambda to check for opportunities for
- // ((0-A) + B) -> B - A
- // (A + (0-B)) -> A - B
- auto CheckFold = [&](Register &MaybeSub, Register &MaybeNewLHS) {
- if (!mi_match(MaybeSub, MRI, m_Neg(m_Reg(NewRHS))))
- return false;
- NewLHS = MaybeNewLHS;
- return true;
- };
-
- return CheckFold(LHS, RHS) || CheckFold(RHS, LHS);
-}
-
bool CombinerHelper::matchCombineInsertVecElts(
MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo) const {
assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT &&
@@ -3193,14 +3051,6 @@ void CombinerHelper::applyCombineInsertVecElts(
MI.eraseFromParent();
}
-void CombinerHelper::applySimplifyAddToSub(
- MachineInstr &MI, std::tuple<Register, Register> &MatchInfo) const {
- Register SubLHS, SubRHS;
- std::tie(SubLHS, SubRHS) = MatchInfo;
- Builder.buildSub(MI.getOperand(0).getReg(), SubLHS, SubRHS);
- MI.eraseFromParent();
-}
-
bool CombinerHelper::matchBinopWithNegInner(Register MInner, Register Other,
unsigned RootOpc, Register Dst,
LLT Ty,
@@ -3917,7 +3767,7 @@ void CombinerHelper::applyXorOfAndWithSameReg(
Observer.changedInstr(MI);
}
-bool CombinerHelper::matchPtrAddZero(MachineInstr &MI) const {
+bool CombinerHelper::matchPtrAddZeroVector(MachineInstr &MI) const {
auto &PtrAdd = cast<GPtrAdd>(MI);
Register DstReg = PtrAdd.getReg(0);
LLT Ty = MRI.getType(DstReg);
@@ -3926,12 +3776,8 @@ bool CombinerHelper::matchPtrAddZero(MachineInstr &MI) const {
if (DL.isNonIntegralAddressSpace(Ty.getScalarType().getAddressSpace()))
return false;
- if (Ty.isPointer()) {
- auto ConstVal = getIConstantVRegVal(PtrAdd.getBaseReg(), MRI);
- return ConstVal && *ConstVal == 0;
- }
-
- assert(Ty.isVector() && "Expecting a vector type");
+ if (!Ty.isVector())
+ return false;
const MachineInstr *VecMI;
if (!mi_match(PtrAdd.getBaseReg(), MRI, m_MInstr(VecMI)))
return false;
@@ -4672,30 +4518,6 @@ bool CombinerHelper::matchOrShiftToFunnelShift(MachineInstr &MI,
return true;
}
-/// Match an FSHL or FSHR that can be combined to a ROTR or ROTL rotate.
-bool CombinerHelper::matchFunnelShiftToRotate(MachineInstr &MI) const {
- unsigned Opc = MI.getOpcode();
- assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);
- Register X = MI.getOperand(1).getReg();
- Register Y = MI.getOperand(2).getReg();
- if (X != Y)
- return false;
- unsigned RotateOpc =
- Opc == TargetOpcode::G_FSHL ? TargetOpcode::G_ROTL : TargetOpcode::G_ROTR;
- return isLegalOrBeforeLegalizer({RotateOpc, {MRI.getType(X), MRI.getType(Y)}});
-}
-
-void CombinerHelper::applyFunnelShiftToRotate(MachineInstr &MI) const {
- unsigned Opc = MI.getOpcode();
- assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);
- bool IsFSHL = Opc == TargetOpcode::G_FSHL;
- Observer.changingInstr(MI);
- MI.setDesc(Builder.getTII().get(IsFSHL ? TargetOpcode::G_ROTL
- : TargetOpcode::G_ROTR));
- MI.removeOperand(2);
- Observer.changedInstr(MI);
-}
-
// Fold (rot x, c) -> (rot x, c % BitSize)
bool CombinerHelper::matchRotateOutOfRange(MachineInstr &MI) const {
assert(MI.getOpcode() == TargetOpcode::G_ROTL ||
@@ -5503,25 +5325,6 @@ bool CombinerHelper::matchNarrowBinopFeedingAnd(
return true;
}
-bool CombinerHelper::matchMulOBy2(MachineInstr &MI,
- BuildFnTy &MatchInfo) const {
- unsigned Opc = MI.getOpcode();
- assert(Opc == TargetOpcode::G_UMULO || Opc == TargetOpcode::G_SMULO);
-
- if (!mi_match(MI.getOperand(3).getReg(), MRI, m_SpecificICstOrSplat(2)))
- return false;
-
- MatchInfo = [=, &MI](MachineIRBuilder &B) {
- Observer.changingInstr(MI);
- unsigned NewOpc = Opc == TargetOpcode::G_UMULO ? TargetOpcode::G_UADDO
- : TargetOpcode::G_SADDO;
- MI.setDesc(Builder.getTII().get(NewOpc));
- MI.getOperand(3).setReg(MI.getOperand(2).getReg());
- Observer.changedInstr(MI);
- };
- return true;
-}
-
bool CombinerHelper::matchMulOBy0(MachineInstr &MI,
BuildFnTy &MatchInfo) const {
// (G_*MULO x, 0) -> 0 + no carry out
@@ -5541,40 +5344,6 @@ bool CombinerHelper::matchMulOBy0(MachineInstr &MI,
return true;
}
-bool CombinerHelper::matchAddEToAddO(MachineInstr &MI,
- BuildFnTy &MatchInfo) const {
- // (G_*ADDE x, y, 0) -> (G_*ADDO x, y)
- // (G_*SUBE x, y, 0) -> (G_*SUBO x, y)
- assert(MI.getOpcode() == TargetOpcode::G_UADDE ||
- MI.getOpcode() == TargetOpcode::G_SADDE ||
- MI.getOpcode() == TargetOpcode::G_USUBE ||
- MI.getOpcode() == TargetOpcode::G_SSUBE);
- if (!mi_match(MI.getOperand(4).getReg(), MRI, m_SpecificICstOrSplat(0)))
- return false;
- MatchInfo = [&](MachineIRBuilder &B) {
- unsigned NewOpcode;
- switch (MI.getOpcode()) {
- case TargetOpcode::G_UADDE:
- NewOpcode = TargetOpcode::G_UADDO;
- break;
- case TargetOpcode::G_SADDE:
- NewOpcode = TargetOpcode::G_SADDO;
- break;
- case TargetOpcode::G_USUBE:
- NewOpcode = TargetOpcode::G_USUBO;
- break;
- case TargetOpcode::G_SSUBE:
- NewOpcode = TargetOpcode::G_SSUBO;
- break;
- }
- Observer.changingInstr(MI);
- MI.setDesc(B.getTII().get(NewOpcode));
- MI.removeOperand(4);
- Observer.changedInstr(MI);
- };
- return true;
-}
-
bool CombinerHelper::matchSubAddSameReg(MachineInstr &MI,
BuildFnTy &MatchInfo) const {
assert(MI.getOpcode() == TargetOpcode::G_SUB);
@@ -6309,46 +6078,10 @@ bool CombinerHelper::matchTruncUSatUToFPTOUISat(MachineInstr &MI,
isLegalOrBeforeLegalizer({TargetOpcode::G_FPTOUI_SAT, {DstTy, SrcTy}});
}
-bool CombinerHelper::matchRedundantNegOperands(MachineInstr &MI,
- BuildFnTy &MatchInfo) const {
- unsigned Opc = MI.getOpcode();
- assert(Opc == TargetOpcode::G_FADD || Opc == TargetOpcode::G_FSUB);
-
- Register Dst = MI.getOperand(0).getReg();
- Register X = MI.getOperand(1).getReg();
- Register Y = MI.getOperand(2).getReg();
- LLT Type = MRI.getType(Dst);
-
- // fold (fadd x, fneg(y)) -> (fsub x, y)
- // fold (fadd fneg(y), x) -> (fsub x, y)
- // G_ADD is commutative so both cases are checked by m_GFAdd
- if (mi_match(Dst, MRI, m_GFAdd(m_Reg(X), m_GFNeg(m_Reg(Y)))) &&
- isLegalOrBeforeLegalizer({TargetOpcode::G_FSUB, {Type}})) {
- Opc = TargetOpcode::G_FSUB;
- }
- /// fold (fsub x, fneg(y)) -> (fadd x, y)
- else if (mi_match(Dst, MRI, m_GFSub(m_Reg(X), m_GFNeg(m_Reg(Y)))) &&
- isLegalOrBeforeLegalizer({TargetOpcode::G_FADD, {Type}})) {
- Opc = TargetOpcode::G_FADD;
- } else
- return false;
-
- MatchInfo = [=, &MI](MachineIRBuilder &B) {
- Observer.changingInstr(MI);
- MI.setDesc(B.getTII().get(Opc));
- MI.getOperand(1).setReg(X);
- MI.getOperand(2).setReg(Y);
- Observer.changedInstr(MI);
- };
- return true;
-}
-
-bool CombinerHelper::matchFsubToFneg(MachineInstr &MI,
- Register &MatchInfo) const {
+bool CombinerHelper::matchFsubToFneg(MachineInstr &MI) const {
assert(MI.getOpcode() == TargetOpcode::G_FSUB);
Register LHS = MI.getOperand(1).getReg();
- MatchInfo = MI.getOperand(2).getReg();
LLT Ty = MRI.getType(MI.getOperand(0).getReg());
const auto LHSCst = Ty.isVector()
@@ -6368,14 +6101,6 @@ bool CombinerHelper::matchFsubToFneg(MachineInstr &MI,
return false;
}
-void CombinerHelper::applyFsubToFneg(MachineInstr &MI,
- Register &MatchInfo) const {
- Register Dst = MI.getOperand(0).getReg();
- Builder.buildFNeg(
- Dst, Builder.buildFCanonicalize(MRI.getType(Dst), MatchInfo).getReg(0));
- eraseInst(MI);
-}
-
/// Checks if \p MI is TargetOpcode::G_FMUL and contractable either
/// due to global flags or MachineInstr flags.
static bool isContractableFMul(MachineInstr &MI, bool AllowFusionGlobally) {
@@ -6978,34 +6703,6 @@ bool CombinerHelper::matchCombineFSubFpExtFNegFMulToFMadOrFMA(
return false;
}
-bool CombinerHelper::matchCombineFMinMaxNaN(MachineInstr &MI,
- unsigned &IdxToPropagate) const {
- bool PropagateNaN;
- switch (MI.getOpcode()) {
- default:
- return false;
- case TargetOpcode::G_FMINNUM:
- case TargetOpcode::G_FMAXNUM:
- PropagateNaN = false;
- break;
- case TargetOpcode::G_FMINIMUM:
- case TargetOpcode::G_FMAXIMUM:
- PropagateNaN = true;
- break;
- }
-
- auto MatchNaN = [&](unsigned Idx) {
- Register MaybeNaNReg = MI.getOperand(Idx).getReg();
- const ConstantFP *MaybeCst = getConstantFPVRegVal(MaybeNaNReg, MRI);
- if (!MaybeCst || !MaybeCst->getValueAPF().isNaN())
- return false;
- IdxToPropagate = PropagateNaN ? Idx : (Idx == 1 ? 2 : 1);
- return true;
- };
-
- return MatchNaN(1) || MatchNaN(2);
-}
-
// Combine multiple FDIVs with the same divisor into multiple FMULs by the
// reciprocal.
// E.g., (a / Y; b / Y;) -> (recip = 1.0 / Y; a * recip; b * recip)
@@ -7427,28 +7124,6 @@ bool CombinerHelper::matchCommuteFPConstantToRHS(MachineInstr &MI) const {
return !mi_match(RHS, MRI, m_GFCstOrSplat(ValAndVReg));
}
-void CombinerHelper::applyCommuteBinOpOperands(MachineInstr &MI) const {
- Observer.changingInstr(MI);
- unsigned LHSOpndIdx = 1;
- unsigned RHSOpndIdx = 2;
- switch (MI.getOpcode()) {
- case TargetOpcode::G_UADDO:
- case TargetOpcode::G_SADDO:
- case TargetOpcode::G_UMULO:
- case TargetOpcode::G_SMULO:
- LHSOpndIdx = 2;
- RHSOpndIdx = 3;
- break;
- default:
- break;
- }
- Register LHSReg = MI.getOperand(LHSOpndIdx).getReg();
- Register RHSReg = MI.getOperand(RHSOpndIdx).getReg();
- MI.getOperand(LHSOpndIdx).setReg(RHSReg);
- MI.getOperand(RHSOpndIdx).setReg(LHSReg);
- Observer.changedInstr(MI);
-}
-
bool CombinerHelper::isOneOrOneSplat(Register Src, bool AllowUndefs) const {
LLT SrcTy = MRI.getType(Src);
if (SrcTy.isFixedVector())
@@ -8168,16 +7843,6 @@ bool CombinerHelper::matchAddOverflow(MachineInstr &MI,
LLT DstTy = MRI.getType(Dst);
LLT CarryTy = MRI.getType(Carry);
- // Fold addo, if the carry is dead -> add, undef.
- if (MRI.use_nodbg_empty(Carry) &&
- isLegalOrBeforeLegalizer({TargetOpcode::G_ADD, {DstTy}})) {
- MatchInfo = [=](MachineIRBuilder &B) {
- B.buildAdd(Dst, LHS, RHS);
- B.buildUndef(Carry);
- };
- return true;
- }
-
// Canonicalize constant to RHS.
if (isConstantOrConstantVectorI(LHS) && !isConstantOrConstantVectorI(RHS)) {
if (IsSigned) {
@@ -8821,19 +8486,6 @@ bool CombinerHelper::matchCtls(MachineInstr &CtlzMI,
return true;
}
-// Fold shr ( add ( ext X, ext Y ), 1 ) -> avgfloor ( x, y )
-// Fold shr ( add ( ext X, ext Y, 1 ), 1 ) -> avgceil ( x, y )
-bool CombinerHelper::matchAVG(MachineInstr &MI, MachineRegisterInfo &MRI,
- Register X, Register Y,
- unsigned TargetOpc) const {
- assert((MI.getOpcode() == TargetOpcode::G_LSHR ||
- MI.getOpcode() == TargetOpcode::G_ASHR) &&
- "Expected G_LSHR/G_ASHR");
-
- LLT XTy = MRI.getType(X);
- return XTy == MRI.getType(Y) && isLegal({TargetOpc, {XTy}});
-}
-
static unsigned getCountZeroPoisonOpcode(const MachineInstr &MI) {
assert((MI.getOpcode() == TargetOpcode::G_CTLZ ||
MI.getOpcode() == TargetOpcode::G_CTTZ) &&
@@ -8861,7 +8513,3 @@ bool CombinerHelper::matchCountZeroToZeroPoison(MachineInstr &MI) const {
LLT SrcTy = MRI.getType(Src);
return isLegalOrBeforeLegalizer({ZPOpc, {DstTy, SrcTy}});
}
-
-void CombinerHelper::applyCountZeroToZeroPoison(MachineInstr &MI) const {
- replaceOpcodeWith(MI, getCountZeroPoisonOpcode(MI));
-}
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelperArtifacts.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelperArtifacts.cpp
deleted file mode 100644
index f2451569d29fd..0000000000000
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelperArtifacts.cpp
+++ /dev/null
@@ -1,85 +0,0 @@
-//===- CombinerHelperArtifacts.cpp-----------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements CombinerHelper for legalization artifacts.
-//
-//===----------------------------------------------------------------------===//
-//
-// G_MERGE_VALUES
-//
-//===----------------------------------------------------------------------===//
-#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
-#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
-#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
-#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
-#include "llvm/CodeGen/GlobalISel/Utils.h"
-#include "llvm/CodeGen/MachineOperand.h"
-#include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/TargetOpcodes.h"
-#include "llvm/Support/Casting.h"
-
-#define DEBUG_TYPE "gi-combiner"
-
-using namespace llvm;
-
-bool CombinerHelper::matchMergeXAndUndef(const MachineInstr &MI,
- BuildFnTy &MatchInfo) const {
- const GMerge *Merge = cast<GMerge>(&MI);
-
- Register Dst = Merge->getReg(0);
- LLT DstTy = MRI.getType(Dst);
- LLT SrcTy = MRI.getType(Merge->getSourceReg(0));
-
- // Otherwise, we would miscompile.
- assert(Merge->getNumSources() == 2 && "Unexpected number of operands");
-
- //
- // %bits_8_15:_(s8) = G_IMPLICIT_DEF
- // %0:_(s16) = G_MERGE_VALUES %bits_0_7:(s8), %bits_8_15:(s8)
- //
- // ->
- //
- // %0:_(s16) = G_ANYEXT %bits_0_7:(s8)
- //
-
- if (!isLegalOrBeforeLegalizer({TargetOpcode::G_ANYEXT, {DstTy, SrcTy}}))
- return false;
-
- MatchInfo = [=](MachineIRBuilder &B) {
- B.buildAnyExt(Dst, Merge->getSourceReg(0));
- };
- return true;
-}
-
-bool CombinerHelper::matchMergeXAndZero(const MachineInstr &MI,
- BuildFnTy &MatchInfo) const {
- const GMerge *Merge = cast<GMerge>(&MI);
-
- Register Dst = Merge->getReg(0);
- LLT DstTy = MRI.getType(Dst);
- LLT SrcTy = MRI.getType(Merge->getSourceReg(0));
-
- // No multi-use check. It is a constant.
-
- //
- // %bits_8_15:_(s8) = G_CONSTANT i8 0
- // %0:_(s16) = G_MERGE_VALUES %bits_0_7:(s8), %bits_8_15:(s8)
- //
- // ->
- //
- // %0:_(s16) = G_ZEXT %bits_0_7:(s8)
- //
-
- if (!isLegalOrBeforeLegalizer({TargetOpcode::G_ZEXT, {DstTy, SrcTy}}))
- return false;
-
- MatchInfo = [=](MachineIRBuilder &B) {
- B.buildZExt(Dst, Merge->getSourceReg(0));
- };
- return true;
-}
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp
index f657c6b49cbb4..e34ca171e36fa 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp
@@ -30,8 +30,7 @@
using namespace llvm;
using namespace MIPatternMatch;
-bool CombinerHelper::matchExtractVectorElement(MachineInstr &MI,
- BuildFnTy &MatchInfo) const {
+bool CombinerHelper::matchExtractVectorElement(MachineInstr &MI) const {
GExtractVectorElement *Extract = cast<GExtractVectorElement>(&MI);
Register Dst = Extract->getReg(0);
@@ -79,11 +78,8 @@ bool CombinerHelper::matchExtractVectorElement(MachineInstr &MI,
// Fold extractVectorElement(Vector, TOOLARGE) -> undef
if (IndexC && VectorTy.isFixedVector() &&
IndexC->uge(VectorTy.getNumElements()) &&
- isLegalOrBeforeLegalizer({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) {
- // For fixed-length vectors, it's invalid to extract out-of-range elements.
- MatchInfo = [=](MachineIRBuilder &B) { B.buildUndef(Dst); };
+ isLegalOrBeforeLegalizer({TargetOpcode::G_IMPLICIT_DEF, {DstTy}}))
return true;
- }
return false;
}
@@ -173,6 +169,12 @@ bool CombinerHelper::matchExtractVectorElementWithBuildVector(
APInt Index = getIConstantFromReg(Extract->getIndexReg(), MRI);
+ // An out-of-bounds constant index extracts undef; it is handled by
+ // extract_vec_elt_out_of_bounds. Bail out here so we never read past the
+ // G_BUILD_VECTOR's sources.
+ if (Index.uge(Build->getNumSources()))
+ return false;
+
// We now know that there is a buildVector def'd on the Vector register and
// the index is const. The combine will succeed.
@@ -234,6 +236,12 @@ bool CombinerHelper::matchExtractVectorElementWithBuildVectorTrunc(
if (!MaybeIndex)
return false;
+ // An out-of-bounds constant index extracts undef; it is handled by
+ // extract_vec_elt_out_of_bounds. Bail out so we never read past the
+ // G_BUILD_VECTOR_TRUNC's sources.
+ if (MaybeIndex->Value.uge(Build->getNumSources()))
+ return false;
+
// We now know that there is a buildVectorTrunc def'd on the Vector register
// and the index is const. The combine will succeed.
@@ -339,8 +347,7 @@ bool CombinerHelper::matchExtractVectorElementWithShuffleVector(
return true;
}
-bool CombinerHelper::matchInsertVectorElementOOB(MachineInstr &MI,
- BuildFnTy &MatchInfo) const {
+bool CombinerHelper::matchInsertVectorElementOOB(MachineInstr &MI) const {
GInsertVectorElement *Insert = cast<GInsertVectorElement>(&MI);
Register Dst = Insert->getReg(0);
@@ -354,10 +361,8 @@ bool CombinerHelper::matchInsertVectorElementOOB(MachineInstr &MI,
getIConstantVRegValWithLookThrough(Index, MRI);
if (MaybeIndex && MaybeIndex->Value.uge(DstTy.getNumElements()) &&
- isLegalOrBeforeLegalizer({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) {
- MatchInfo = [=](MachineIRBuilder &B) { B.buildUndef(Dst); };
+ isLegalOrBeforeLegalizer({TargetOpcode::G_IMPLICIT_DEF, {DstTy}}))
return true;
- }
return false;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
index 2f0ff66668f2b..82d5d15482c4e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
@@ -228,8 +228,7 @@ def fmin_ieee_to_minmax3 : minmax_to_minmax3_opcodes<G_FMINNUM_IEEE>;
class combine_binop_s64_with_s32_mask<Instruction opcode> : GICombineRule<
(defs root:$dst),
(match (opcode $dst, i64:$x, i64:$y):$dst,
- [{ return Helper.matchConstantIs32BitMask(${x}.getReg()) ||
- Helper.matchConstantIs32BitMask(${y}.getReg()); }]),
+ [{ return Helper.matchBinopWith32BitMask(${x}.getReg(), ${y}.getReg()); }]),
(apply (G_UNMERGE_VALUES i32:$x_lo, i32:$x_hi, $x),
(G_UNMERGE_VALUES i32:$y_lo, i32:$y_hi, $y),
(opcode i32:$lo, $x_lo, $y_lo),
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
index 245a3930fe1c8..b06d46e105416 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
@@ -572,3 +572,14 @@ bool AMDGPUCombinerHelper::matchConstantIs32BitMask(Register Reg) const {
// Check if low 32 bits or high 32 bits are all ones.
return MaskLen >= 32 && ((MaskIdx == 0) || (MaskIdx == 64 - MaskLen));
}
+
+bool AMDGPUCombinerHelper::matchBinopWith32BitMask(Register X,
+ Register Y) const {
+ // If both operands are constants, constant_fold_binop can fold the whole
+ // binop to a single constant, which is strictly better than narrowing it
+ // into two s32 ops. Bail so the fold is not preempted.
+ if (getIConstantVRegValWithLookThrough(X, MRI) &&
+ getIConstantVRegValWithLookThrough(Y, MRI))
+ return false;
+ return matchConstantIs32BitMask(X) || matchConstantIs32BitMask(Y);
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h
index 7e910dc23367f..bff9efb3621eb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h
@@ -49,6 +49,12 @@ class AMDGPUCombinerHelper : public CombinerHelper {
bool matchConstantIs32BitMask(Register Reg) const;
+ /// Match an s64 and/or where one operand is a 32-bit mask (see
+ /// matchConstantIs32BitMask), used to narrow it into two s32 ops. Bails when
+ /// both operands are constants, since folding the binop to a single constant
+ /// is strictly better than narrowing.
+ bool matchBinopWith32BitMask(Register X, Register Y) const;
+
/// fmin_legacy/fmax_legacy select s1 on NaN, and on a +0.0/-0.0 tie (s1 for
/// min, s0 for max). Returns true if that tie cannot be observed: nsz on
/// \p MI, or a known non-logical-zero \p LHS or \p RHS.
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-adde-to-addo.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-adde-to-addo.mir
new file mode 100644
index 0000000000000..b2ca725ba8224
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-adde-to-addo.mir
@@ -0,0 +1,155 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner -global-isel %s -o - | FileCheck %s
+
+# (G_*ADDE x, y, 0) -> (G_*ADDO x, y), including splat-zero carry-in.
+
+---
+name: uadde_zero
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: uadde_zero
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %x:_(i32) = COPY $w0
+ ; CHECK-NEXT: %y:_(i32) = COPY $w1
+ ; CHECK-NEXT: %add:_(i32), %o:_(i1) = G_UADDO %x, %y
+ ; CHECK-NEXT: %o_wide:_(i32) = G_ZEXT %o(i1)
+ ; CHECK-NEXT: $w0 = COPY %add(i32)
+ ; CHECK-NEXT: $w1 = COPY %o_wide(i32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %x:_(i32) = COPY $w0
+ %y:_(i32) = COPY $w1
+ %zero:_(i1) = G_CONSTANT i1 0
+ %add:_(i32), %o:_(i1) = G_UADDE %x, %y, %zero
+ %o_wide:_(i32) = G_ZEXT %o(i1)
+ $w0 = COPY %add(i32)
+ $w1 = COPY %o_wide
+ RET_ReallyLR implicit $w0
+...
+---
+name: sadde_zero
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: sadde_zero
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %x:_(i32) = COPY $w0
+ ; CHECK-NEXT: %y:_(i32) = COPY $w1
+ ; CHECK-NEXT: %add:_(i32), %o:_(i1) = G_SADDO %x, %y
+ ; CHECK-NEXT: %o_wide:_(i32) = G_ZEXT %o(i1)
+ ; CHECK-NEXT: $w0 = COPY %add(i32)
+ ; CHECK-NEXT: $w1 = COPY %o_wide(i32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %x:_(i32) = COPY $w0
+ %y:_(i32) = COPY $w1
+ %zero:_(i1) = G_CONSTANT i1 0
+ %add:_(i32), %o:_(i1) = G_SADDE %x, %y, %zero
+ %o_wide:_(i32) = G_ZEXT %o(i1)
+ $w0 = COPY %add(i32)
+ $w1 = COPY %o_wide
+ RET_ReallyLR implicit $w0
+...
+---
+name: usube_zero
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: usube_zero
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %x:_(i32) = COPY $w0
+ ; CHECK-NEXT: %y:_(i32) = COPY $w1
+ ; CHECK-NEXT: %sub:_(i32), %o:_(i1) = G_USUBO %x, %y
+ ; CHECK-NEXT: %o_wide:_(i32) = G_ZEXT %o(i1)
+ ; CHECK-NEXT: $w0 = COPY %sub(i32)
+ ; CHECK-NEXT: $w1 = COPY %o_wide(i32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %x:_(i32) = COPY $w0
+ %y:_(i32) = COPY $w1
+ %zero:_(i1) = G_CONSTANT i1 0
+ %sub:_(i32), %o:_(i1) = G_USUBE %x, %y, %zero
+ %o_wide:_(i32) = G_ZEXT %o(i1)
+ $w0 = COPY %sub(i32)
+ $w1 = COPY %o_wide
+ RET_ReallyLR implicit $w0
+...
+---
+name: ssube_zero
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: ssube_zero
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %x:_(i32) = COPY $w0
+ ; CHECK-NEXT: %y:_(i32) = COPY $w1
+ ; CHECK-NEXT: %sub:_(i32), %o:_(i1) = G_SSUBO %x, %y
+ ; CHECK-NEXT: %o_wide:_(i32) = G_ZEXT %o(i1)
+ ; CHECK-NEXT: $w0 = COPY %sub(i32)
+ ; CHECK-NEXT: $w1 = COPY %o_wide(i32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %x:_(i32) = COPY $w0
+ %y:_(i32) = COPY $w1
+ %zero:_(i1) = G_CONSTANT i1 0
+ %sub:_(i32), %o:_(i1) = G_SSUBE %x, %y, %zero
+ %o_wide:_(i32) = G_ZEXT %o(i1)
+ $w0 = COPY %sub(i32)
+ $w1 = COPY %o_wide
+ RET_ReallyLR implicit $w0
+...
+---
+name: vector_splat_zero
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $d0, $d1
+ ; CHECK-LABEL: name: vector_splat_zero
+ ; CHECK: liveins: $d0, $d1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %x:_(<2 x i32>) = COPY $d0
+ ; CHECK-NEXT: %y:_(<2 x i32>) = COPY $d1
+ ; CHECK-NEXT: %add:_(<2 x i32>), %o:_(<2 x i1>) = G_UADDO %x, %y
+ ; CHECK-NEXT: %o_ext:_(<2 x i32>) = G_ANYEXT %o(<2 x i1>)
+ ; CHECK-NEXT: $d0 = COPY %o_ext(<2 x i32>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $d0
+ %x:_(<2 x i32>) = COPY $d0
+ %y:_(<2 x i32>) = COPY $d1
+ %zero:_(i1) = G_CONSTANT i1 0
+ %splat_zero:_(<2 x i1>) = G_BUILD_VECTOR %zero(i1), %zero(i1)
+ %add:_(<2 x i32>), %o:_(<2 x i1>) = G_UADDE %x, %y, %splat_zero
+ %o_ext:_(<2 x i32>) = G_ANYEXT %o(<2 x i1>)
+ $d0 = COPY %o_ext(<2 x i32>)
+ RET_ReallyLR implicit $d0
+...
+---
+name: dont_combine_nonzero_carry
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $w0, $w1
+ ; CHECK-LABEL: name: dont_combine_nonzero_carry
+ ; CHECK: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %x:_(i32) = COPY $w0
+ ; CHECK-NEXT: %y:_(i32) = COPY $w1
+ ; CHECK-NEXT: %one:_(i1) = G_CONSTANT i1 true
+ ; CHECK-NEXT: %add:_(i32), %o:_(i1) = G_UADDE %x, %y, %one
+ ; CHECK-NEXT: %o_wide:_(i32) = G_ZEXT %o(i1)
+ ; CHECK-NEXT: $w0 = COPY %add(i32)
+ ; CHECK-NEXT: $w1 = COPY %o_wide(i32)
+ ; CHECK-NEXT: RET_ReallyLR implicit $w0
+ %x:_(i32) = COPY $w0
+ %y:_(i32) = COPY $w1
+ %one:_(i1) = G_CONSTANT i1 1
+ %add:_(i32), %o:_(i1) = G_UADDE %x, %y, %one
+ %o_wide:_(i32) = G_ZEXT %o(i1)
+ $w0 = COPY %add(i32)
+ $w1 = COPY %o_wide
+ RET_ReallyLR implicit $w0
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-mulo-with-2.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-mulo-with-2.mir
index 5271f18caffe9..9e6de95857b58 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-mulo-with-2.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-mulo-with-2.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple aarch64 -debugify-and-strip-all-safe -run-pass=aarch64-prelegalizer-combiner --aarch64prelegalizercombiner-only-enable-rule="mulo_by_2" -global-isel -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple aarch64 -debugify-and-strip-all-safe -run-pass=aarch64-prelegalizer-combiner --aarch64prelegalizercombiner-only-enable-rule="mulo_by_2_umulo,mulo_by_2_smulo" -global-isel -verify-machineinstrs %s -o - | FileCheck %s
# REQUIRES: asserts
...
---
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-shift-immed-mismatch-crash.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-shift-immed-mismatch-crash.mir
index da915b6daadbc..8aa423f36ce3c 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-shift-immed-mismatch-crash.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-shift-immed-mismatch-crash.mir
@@ -12,17 +12,17 @@ body: |
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: liveins: $x0
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(i1) = G_IMPLICIT_DEF
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 9
- ; CHECK-NEXT: G_BRCOND [[DEF]](i1), %bb.2
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(i1) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: G_BRCOND [[DEF1]](i1), %bb.2
; CHECK-NEXT: G_BR %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors:
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[DEF1]](p0) :: (load (i32) from `ptr undef`, align 8)
+ ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[DEF]](p0) :: (load (i32) from `ptr undef`, align 8)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = nsw G_SHL [[LOAD]], [[C1]](i32)
; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i32) = nsw G_MUL [[SHL]], [[C]]
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-undef.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-undef.mir
index bc65891cec7ce..170de81c7a629 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-undef.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-undef.mir
@@ -34,8 +34,8 @@ body: |
; CHECK-LABEL: name: delete_add_undef
; CHECK: liveins: $x0
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(i64) = G_IMPLICIT_DEF
- ; CHECK-NEXT: $x0 = COPY [[DEF]](i64)
+ ; CHECK-NEXT: %add:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $x0 = COPY %add(i64)
; CHECK-NEXT: RET_ReallyLR
%0:_(i64) = COPY $x0
%1:_(i64) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-addo-zero.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-addo-zero.mir
index aa782e6aa7405..e086a03af2f50 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-addo-zero.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-addo-zero.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple aarch64 -debugify-and-strip-all-safe -run-pass=aarch64-prelegalizer-combiner --aarch64prelegalizercombiner-only-enable-rule="match_addos" -global-isel -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple aarch64 -debugify-and-strip-all-safe -run-pass=aarch64-prelegalizer-combiner --aarch64prelegalizercombiner-only-enable-rule="match_addos_saddo,match_addos_uaddo" -global-isel -verify-machineinstrs %s -o - | FileCheck %s
# REQUIRES: asserts
# (G_*ADDO x, 0) -> x + no carry
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-mulo-zero.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-mulo-zero.mir
index d6e6e4c365357..6f1a21ab1847f 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-mulo-zero.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-mulo-zero.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple aarch64 -debugify-and-strip-all-safe -run-pass=aarch64-prelegalizer-combiner --aarch64prelegalizercombiner-only-enable-rule="mulo_by_0" -global-isel -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple aarch64 -debugify-and-strip-all-safe -run-pass=aarch64-prelegalizer-combiner --aarch64prelegalizercombiner-only-enable-rule="mulo_by_0_umulo,mulo_by_0_smulo" -global-isel -verify-machineinstrs %s -o - | FileCheck %s
# REQUIRES: asserts
# (G_*MULO x, 0) -> 0 + no carry out
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-prop-extends-phi.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-prop-extends-phi.mir
index b146b56f7b0d7..027090c1250c7 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-prop-extends-phi.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-prop-extends-phi.mir
@@ -126,8 +126,8 @@ body: |
; CHECK-NEXT: liveins: $x0, $q0, $q1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %ptr:_(p0) = COPY $x0
- ; CHECK-NEXT: %cmp:_(i1) = G_IMPLICIT_DEF
- ; CHECK-NEXT: G_BRCOND %cmp(i1), %bb.2
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(i1) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: G_BRCOND [[DEF]](i1), %bb.2
; CHECK-NEXT: G_BR %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
diff --git a/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll b/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll
index 93c22573c0511..128b67663bf04 100644
--- a/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/neon-shuffle-vector-tbl.ll
@@ -559,7 +559,7 @@ define <8 x i8> @no_shuffle_only_some_and_constants(<8 x i8> %src, <8 x i8> %mas
; CHECK-GI-NEXT: mov x8, sp
; CHECK-GI-NEXT: str d0, [sp]
; CHECK-GI-NEXT: and w9, w9, #0x7
-; CHECK-GI-NEXT: and x9, x9, #0x7
+; CHECK-GI-NEXT: and x9, x9, #0xff
; CHECK-GI-NEXT: lsl x11, x9, #1
; CHECK-GI-NEXT: sub x9, x11, x9
; CHECK-GI-NEXT: ldr b2, [x8, x9]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-binop-s64-with-s32-mask.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-binop-s64-with-s32-mask.mir
index fb3bf9db3881b..96cc7704f49e9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-binop-s64-with-s32-mask.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-binop-s64-with-s32-mask.mir
@@ -32,8 +32,8 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](i32), [[UV1]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
@@ -222,8 +222,8 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[COPY]](s64)
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[TRUNC]](i32), [[C]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
@@ -264,8 +264,8 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](i32), [[UV1]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-extract-vector-load.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-extract-vector-load.mir
index d4b1b75fa82a9..c595f8b085739 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-extract-vector-load.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-extract-vector-load.mir
@@ -8,8 +8,9 @@ tracksRegLiveness: true
body: |
bb.0:
; CHECK-LABEL: name: test_ptradd_crash__offset_smaller
- ; CHECK: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 12
- ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[C]](p1) :: (load (s32), addrspace 1)
+ ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
+ ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p1) = G_INTTOPTR [[C]](s64)
+ ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[INTTOPTR]](p1) :: (load (s32), addrspace 1)
; CHECK-NEXT: $sgpr0 = COPY [[LOAD]](s32)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0
%1:_(p1) = G_CONSTANT i64 0
@@ -27,8 +28,9 @@ tracksRegLiveness: true
body: |
bb.0:
; CHECK-LABEL: name: test_ptradd_crash__offset_wider
- ; CHECK: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 12
- ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[C]](p1) :: (load (s32), addrspace 1)
+ ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
+ ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p1) = G_INTTOPTR [[C]](s64)
+ ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[INTTOPTR]](p1) :: (load (s32), addrspace 1)
; CHECK-NEXT: $sgpr0 = COPY [[LOAD]](s32)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0
%1:_(p1) = G_CONSTANT i64 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fold-binop-into-select.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fold-binop-into-select.mir
index 71cc602bb1d22..7e3392f0297d7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fold-binop-into-select.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fold-binop-into-select.mir
@@ -726,8 +726,8 @@ body: |
; CHECK-NEXT: %cond:_(s1) = G_ICMP intpred(eq), %reg(s32), %zero
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 50
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 21
- ; CHECK-NEXT: %sdiv:_(s32) = G_SELECT %cond(s1), [[C]], [[C1]]
- ; CHECK-NEXT: S_ENDPGM 0, implicit %sdiv(s32)
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT %cond(s1), [[C]], [[C1]]
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[SELECT]](s32)
%reg:_(s32) = COPY $vgpr0
%zero:_(s32) = G_CONSTANT i32 0
%cond:_(s1) = G_ICMP intpred(eq), %reg, %zero
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-foldable-fneg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-foldable-fneg.mir
index 92abbee19bae1..74006d590e48b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-foldable-fneg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-foldable-fneg.mir
@@ -143,8 +143,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
- ; CHECK-NEXT: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[COPY]]
- ; CHECK-NEXT: [[FSUB:%[0-9]+]]:_(s32) = nsz G_FSUB [[FNEG]], [[COPY1]]
+ ; CHECK-NEXT: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[COPY1]]
+ ; CHECK-NEXT: [[FSUB:%[0-9]+]]:_(s32) = nsz G_FSUB [[FNEG]], [[COPY]]
; CHECK-NEXT: $vgpr0 = COPY [[FSUB]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fsub-fneg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fsub-fneg.mir
index a27a231d8dce2..3aaa49dcbe6e6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fsub-fneg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fsub-fneg.mir
@@ -14,7 +14,7 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: %input:_(s16) = G_TRUNC [[COPY]](s32)
; CHECK-NEXT: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG %input
- ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[FNEG]]
+ ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = nsz G_FCANONICALIZE [[FNEG]]
; CHECK-NEXT: %res:_(s32) = G_ANYEXT [[FCANONICALIZE]](s16)
; CHECK-NEXT: $vgpr0 = COPY %res(s32)
%0:_(s32) = COPY $vgpr0
@@ -85,7 +85,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %input:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG %input
- ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FNEG]]
+ ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s32) = nsz G_FCANONICALIZE [[FNEG]]
; CHECK-NEXT: $vgpr0 = COPY [[FCANONICALIZE]](s32)
%input:_(s32) = COPY $vgpr0
%cst:_(s32) = G_FCONSTANT float 0.0
@@ -145,7 +145,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %input:_(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG %input
- ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s64) = G_FCANONICALIZE [[FNEG]]
+ ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s64) = nsz G_FCANONICALIZE [[FNEG]]
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[FCANONICALIZE]](s64)
%input:_(s64) = COPY $vgpr0_vgpr1
%cst:_(s64) = G_FCONSTANT double 0.0
@@ -205,7 +205,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %input:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[FNEG:%[0-9]+]]:_(<4 x s16>) = G_FNEG %input
- ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<4 x s16>) = G_FCANONICALIZE [[FNEG]]
+ ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<4 x s16>) = nsz G_FCANONICALIZE [[FNEG]]
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[FCANONICALIZE]](<4 x s16>)
%input:_(<4 x s16>) = COPY $vgpr0_vgpr1
%cst:_(s16) = G_FCONSTANT half 0.0
@@ -269,7 +269,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %input:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[FNEG:%[0-9]+]]:_(<4 x s32>) = G_FNEG %input
- ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<4 x s32>) = G_FCANONICALIZE [[FNEG]]
+ ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<4 x s32>) = nsz G_FCANONICALIZE [[FNEG]]
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[FCANONICALIZE]](<4 x s32>)
%input:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%cst:_(s32) = G_FCONSTANT float 0.0
@@ -333,7 +333,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %input:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[FNEG:%[0-9]+]]:_(<4 x s32>) = G_FNEG %input
- ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<4 x s32>) = G_FCANONICALIZE [[FNEG]]
+ ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<4 x s32>) = nsz G_FCANONICALIZE [[FNEG]]
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[FCANONICALIZE]](<4 x s32>)
%input:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%cst:_(s32) = G_FCONSTANT float 0.0
@@ -355,7 +355,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %input:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[FNEG:%[0-9]+]]:_(<2 x s64>) = G_FNEG %input
- ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x s64>) = G_FCANONICALIZE [[FNEG]]
+ ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(<2 x s64>) = nsz G_FCANONICALIZE [[FNEG]]
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[FCANONICALIZE]](<2 x s64>)
%input:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%cst:_(s64) = G_FCONSTANT double 0.0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
index 7e83c0d02b43f..9d017fd71ef77 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
@@ -348,14 +348,14 @@ body: |
; GFX6-LABEL: name: do_not_shl_s32_zero_by_16_from_zext_s16
; GFX6: liveins: $vgpr0
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: %shl:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: $vgpr0 = COPY %shl(s32)
+ ; GFX6-NEXT: %extend:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: $vgpr0 = COPY %extend(s32)
;
; GFX9-LABEL: name: do_not_shl_s32_zero_by_16_from_zext_s16
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: %shl:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: $vgpr0 = COPY %shl(s32)
+ ; GFX9-NEXT: %extend:_(s32) = G_CONSTANT i32 0
+ ; GFX9-NEXT: $vgpr0 = COPY %extend(s32)
%zero:_(s16) = G_CONSTANT i16 0
%extend:_(s32) = G_ZEXT %zero:_(s16)
%shiftamt:_(s16) = G_CONSTANT i16 16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
index 1f79f289ac719..5739d01e61d95 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
@@ -37,9 +37,9 @@ define amdgpu_ps ptr addrspace(8) @basic_raw_buffer(ptr inreg %p) {
; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO -6629298651489370112
; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[S_MOV_B]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 9
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -536870912
; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
- ; CHECK45-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 9
; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE1]], [[S_MOV_B64_]], implicit-def dead $scc
; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub0
; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub1
@@ -97,9 +97,9 @@ define amdgpu_ps ptr addrspace(8) @large_num_records_raw_buffer(ptr inreg %p) {
; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO -6629298651489370112
; CHECK45-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE]], [[S_MOV_B]], implicit-def dead $scc
+ ; CHECK45-NEXT: [[S_MOV_B1:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 33554441
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -536870912
; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
- ; CHECK45-NEXT: [[S_MOV_B1:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 33554441
; CHECK45-NEXT: [[S_OR_B64_1:%[0-9]+]]:sreg_64 = S_OR_B64 [[REG_SEQUENCE1]], [[S_MOV_B1]], implicit-def dead $scc
; CHECK45-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub0
; CHECK45-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_OR_B64_]].sub1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
index 51251379963f1..359154a2ae248 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
@@ -18,45 +18,42 @@ define void @test(ptr %p) {
; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
; CHECK-NEXT: v_readfirstlane_b32 s4, v2
; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: s_mul_hi_u32 s6, s4, s6
-; CHECK-NEXT: s_add_i32 s4, s4, s6
+; CHECK-NEXT: s_mul_hi_u32 s7, s4, s6
+; CHECK-NEXT: s_add_i32 s4, s4, s7
; CHECK-NEXT: s_mul_hi_u32 s4, s4, s8
-; CHECK-NEXT: s_mov_b32 s6, 2
-; CHECK-NEXT: s_add_i32 s4, s4, s6
-; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: s_mov_b32 s7, 2
+; CHECK-NEXT: s_add_i32 s4, s4, s7
+; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: v_mov_b32_e32 v2, s7
; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
-; CHECK-NEXT: v_readfirstlane_b32 s6, v2
+; CHECK-NEXT: v_readfirstlane_b32 s7, v2
+; CHECK-NEXT: s_mul_hi_u32 s9, s7, s6
+; CHECK-NEXT: s_add_i32 s7, s7, s9
+; CHECK-NEXT: s_mul_hi_u32 s7, s7, s8
+; CHECK-NEXT: s_mov_b32 s9, 2
+; CHECK-NEXT: s_add_i32 s9, s7, s9
; CHECK-NEXT: s_mov_b32 s7, 0
-; CHECK-NEXT: s_mul_hi_u32 s7, s6, s7
-; CHECK-NEXT: s_add_i32 s6, s6, s7
-; CHECK-NEXT: s_mul_hi_u32 s6, s6, s8
-; CHECK-NEXT: s_mov_b32 s7, 2
-; CHECK-NEXT: s_add_i32 s9, s6, s7
-; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: v_mov_b32_e32 v2, s7
; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
-; CHECK-NEXT: v_readfirstlane_b32 s6, v2
-; CHECK-NEXT: s_mov_b32 s7, 0
-; CHECK-NEXT: s_mul_hi_u32 s7, s6, s7
-; CHECK-NEXT: s_add_i32 s6, s6, s7
-; CHECK-NEXT: s_mul_hi_u32 s6, s6, s8
-; CHECK-NEXT: s_mov_b32 s7, 2
-; CHECK-NEXT: s_add_i32 s7, s6, s7
-; CHECK-NEXT: s_mov_b32 s6, 0
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: v_readfirstlane_b32 s7, v2
+; CHECK-NEXT: s_mul_hi_u32 s10, s7, s6
+; CHECK-NEXT: s_add_i32 s7, s7, s10
+; CHECK-NEXT: s_mul_hi_u32 s7, s7, s8
+; CHECK-NEXT: s_mov_b32 s10, 2
+; CHECK-NEXT: s_add_i32 s7, s7, s10
+; CHECK-NEXT: s_mov_b32 s10, 0
+; CHECK-NEXT: v_mov_b32_e32 v2, s10
; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
; CHECK-NEXT: v_readfirstlane_b32 s5, v2
-; CHECK-NEXT: s_mov_b32 s6, 0
; CHECK-NEXT: s_mul_hi_u32 s6, s5, s6
; CHECK-NEXT: s_add_i32 s5, s5, s6
; CHECK-NEXT: s_mul_hi_u32 s5, s5, s8
diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/GlobalISel/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/GlobalISel/BUILD.gn
index 49e87ed5a41a6..6cca5b7f7073b 100644
--- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/GlobalISel/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/GlobalISel/BUILD.gn
@@ -18,7 +18,6 @@ static_library("GlobalISel") {
"CallLowering.cpp",
"Combiner.cpp",
"CombinerHelper.cpp",
- "CombinerHelperArtifacts.cpp",
"CombinerHelperCasts.cpp",
"CombinerHelperCompares.cpp",
"CombinerHelperVectorOps.cpp",
More information about the llvm-commits
mailing list