[llvm] [X86][APX] Add VirtRegMap to non stack foldMemoryOperand too (PR #193423)
Phoebe Wang via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 22 00:09:23 PDT 2026
https://github.com/phoebewang created https://github.com/llvm/llvm-project/pull/193423
We need to query mapped physical register through VirtRegMap.
Fixes: https://godbolt.org/z/1KGj3aYeP
>From 222e6485fa331422b9322971bdbe77c748a108af Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang at intel.com>
Date: Wed, 22 Apr 2026 15:01:23 +0800
Subject: [PATCH] [X86][APX] Add VirtRegMap to non stack foldMemoryOperand too
We need to query mapped physical register through VirtRegMap.
Fixes: https://godbolt.org/z/1KGj3aYeP
---
llvm/include/llvm/CodeGen/TargetInstrInfo.h | 6 +-
llvm/lib/CodeGen/InlineSpiller.cpp | 2 +-
llvm/lib/CodeGen/LiveRangeEdit.cpp | 2 +-
llvm/lib/CodeGen/TargetInstrInfo.cpp | 11 +-
llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 2 +-
llvm/lib/Target/RISCV/RISCVInstrInfo.h | 11 +-
llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp | 2 +-
llvm/lib/Target/SystemZ/SystemZInstrInfo.h | 11 +-
llvm/lib/Target/X86/X86InstrInfo.cpp | 6 +-
llvm/lib/Target/X86/X86InstrInfo.h | 11 +-
.../CodeGen/X86/apx/memfold-no-physreg.ll | 270 ++++++++++++++++++
11 files changed, 307 insertions(+), 27 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/apx/memfold-no-physreg.ll
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 49562f16bf371..695ed058bc119 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -1266,7 +1266,8 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
/// store from / to any address, not just from a specific stack slot.
MachineInstr *foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops,
MachineInstr &LoadMI, MachineInstr *&CopyMI,
- LiveIntervals *LIS = nullptr) const;
+ LiveIntervals *LIS = nullptr,
+ VirtRegMap *VRM = nullptr) const;
/// This function defines the logic to lower COPY instruction to
/// target specific instruction(s).
@@ -1460,7 +1461,8 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
virtual MachineInstr *foldMemoryOperandImpl(
MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI,
- MachineInstr *&CopyMI, LiveIntervals *LIS = nullptr) const {
+ MachineInstr *&CopyMI, LiveIntervals *LIS = nullptr,
+ VirtRegMap *VRM = nullptr) const {
return nullptr;
}
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index ea062282b97b2..5afcea52b7749 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -1019,7 +1019,7 @@ foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>> Ops,
MachineInstr *CopyMI = nullptr;
MachineInstr *FoldMI =
LoadMI
- ? TII.foldMemoryOperand(*MI, FoldOps, *LoadMI, CopyMI, &LIS)
+ ? TII.foldMemoryOperand(*MI, FoldOps, *LoadMI, CopyMI, &LIS, &VRM)
: TII.foldMemoryOperand(*MI, FoldOps, StackSlot, CopyMI, &LIS, &VRM);
if (!FoldMI) {
// Re-tie operands.
diff --git a/llvm/lib/CodeGen/LiveRangeEdit.cpp b/llvm/lib/CodeGen/LiveRangeEdit.cpp
index edff8c8600188..6208155951b4d 100644
--- a/llvm/lib/CodeGen/LiveRangeEdit.cpp
+++ b/llvm/lib/CodeGen/LiveRangeEdit.cpp
@@ -153,7 +153,7 @@ bool LiveRangeEdit::foldAsLoad(LiveInterval *LI,
MachineInstr *CopyMI = nullptr;
MachineInstr *FoldMI =
- TII.foldMemoryOperand(*UseMI, Ops, *DefMI, CopyMI, &LIS);
+ TII.foldMemoryOperand(*UseMI, Ops, *DefMI, CopyMI, &LIS, VRM);
if (!FoldMI)
return false;
LLVM_DEBUG(dbgs() << " folded: " << *FoldMI);
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index ba836df02048c..a5434e5dfe8a3 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -804,11 +804,10 @@ MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
return &*--Pos;
}
-MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
- ArrayRef<unsigned> Ops,
- MachineInstr &LoadMI,
- MachineInstr *&CopyMI,
- LiveIntervals *LIS) const {
+MachineInstr *
+TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops,
+ MachineInstr &LoadMI, MachineInstr *&CopyMI,
+ LiveIntervals *LIS, VirtRegMap *VRM) const {
assert(LoadMI.canFoldAsLoad() && "LoadMI isn't foldable!");
#ifndef NDEBUG
for (unsigned OpIdx : Ops)
@@ -834,7 +833,7 @@ MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
return foldInlineAsmMemOperand(MI, Ops, FrameIndex, *this);
} else {
// Ask the target to do the actual folding.
- NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, LoadMI, CopyMI, LIS);
+ NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, LoadMI, CopyMI, LIS, VRM);
}
if (!NewMI)
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 6957c6ad14c1b..7880a2e903042 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -957,7 +957,7 @@ static unsigned getLoadPredicatedOpcode(unsigned Opcode) {
MachineInstr *RISCVInstrInfo::foldMemoryOperandImpl(
MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI,
- MachineInstr *&CopyMI, LiveIntervals *LIS) const {
+ MachineInstr *&CopyMI, LiveIntervals *LIS, VirtRegMap *VRM) const {
// For now, only handle RISCV::PseudoCCMOVGPR.
if (MI.getOpcode() != RISCV::PseudoCCMOVGPR)
return nullptr;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 80e7801c1bcb3..2e146c6a9f7c5 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -134,10 +134,13 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
LiveIntervals *LIS = nullptr,
VirtRegMap *VRM = nullptr) const override;
- MachineInstr *foldMemoryOperandImpl(
- MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
- MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI,
- MachineInstr *&CopyMI, LiveIntervals *LIS = nullptr) const override;
+ MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
+ ArrayRef<unsigned> Ops,
+ MachineBasicBlock::iterator InsertPt,
+ MachineInstr &LoadMI,
+ MachineInstr *&CopyMI,
+ LiveIntervals *LIS = nullptr,
+ VirtRegMap *VRM = nullptr) const override;
// Materializes the given integer Val into DstReg.
void movImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index 5d85a64844592..97c10b3c1f422 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -1558,7 +1558,7 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(
MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(
MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI,
- MachineInstr *&CopyMI, LiveIntervals *LIS) const {
+ MachineInstr *&CopyMI, LiveIntervals *LIS, VirtRegMap *VRM) const {
MachineRegisterInfo *MRI = &MF.getRegInfo();
MachineBasicBlock *MBB = MI.getParent();
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.h b/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
index 9fbd8e9a28d1d..0101958e69941 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
@@ -298,10 +298,13 @@ class SystemZInstrInfo : public SystemZGenInstrInfo {
int FrameIndex, MachineInstr *&CopyMI,
LiveIntervals *LIS = nullptr,
VirtRegMap *VRM = nullptr) const override;
- MachineInstr *foldMemoryOperandImpl(
- MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
- MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI,
- MachineInstr *&CopyMI, LiveIntervals *LIS = nullptr) const override;
+ MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
+ ArrayRef<unsigned> Ops,
+ MachineBasicBlock::iterator InsertPt,
+ MachineInstr &LoadMI,
+ MachineInstr *&CopyMI,
+ LiveIntervals *LIS = nullptr,
+ VirtRegMap *VRM = nullptr) const override;
bool expandPostRAPseudo(MachineInstr &MBBI) const override;
bool reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const
override;
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 3656f9ba68c53..2b6cffe9bf4aa 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -8176,7 +8176,7 @@ static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI,
MachineInstr *X86InstrInfo::foldMemoryOperandImpl(
MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI,
- MachineInstr *&CopyMI, LiveIntervals *LIS) const {
+ MachineInstr *&CopyMI, LiveIntervals *LIS, VirtRegMap *VRM) const {
// If LoadMI is a masked load, check MI having the same mask.
const MCInstrDesc &MCID = get(LoadMI.getOpcode());
@@ -8228,8 +8228,8 @@ MachineInstr *X86InstrInfo::foldMemoryOperandImpl(
if (isLoadFromStackSlot(LoadMI, FrameIndex)) {
if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF))
return nullptr;
- return foldMemoryOperandImpl(MF, MI, Ops, InsertPt, FrameIndex, CopyMI,
- LIS);
+ return foldMemoryOperandImpl(MF, MI, Ops, InsertPt, FrameIndex, CopyMI, LIS,
+ VRM);
}
// Check switch flag
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index 1c37fcd7e73b9..a186655ef2ecf 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -501,10 +501,13 @@ class X86InstrInfo final : public X86GenInstrInfo {
/// Same as the previous version except it allows folding of any load and
/// store from / to any address, not just from a specific stack slot.
- MachineInstr *foldMemoryOperandImpl(
- MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
- MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI,
- MachineInstr *&CopyMI, LiveIntervals *LIS = nullptr) const override;
+ MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
+ ArrayRef<unsigned> Ops,
+ MachineBasicBlock::iterator InsertPt,
+ MachineInstr &LoadMI,
+ MachineInstr *&CopyMI,
+ LiveIntervals *LIS = nullptr,
+ VirtRegMap *VRM = nullptr) const override;
bool
unfoldMemoryOperand(MachineFunction &MF, MachineInstr &MI, Register Reg,
diff --git a/llvm/test/CodeGen/X86/apx/memfold-no-physreg.ll b/llvm/test/CodeGen/X86/apx/memfold-no-physreg.ll
new file mode 100644
index 0000000000000..740b321165b63
--- /dev/null
+++ b/llvm/test/CodeGen/X86/apx/memfold-no-physreg.ll
@@ -0,0 +1,270 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-pc-windows-gnu -mattr=egpr,ndd,ccmp,avx512vl -verify-machineinstrs | FileCheck %s
+
+define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr %p2, ptr %p3, ptr %p4, ptr %p5, ptr %p6, ptr %p7, ptr %p8, ptr %p9, ptr %p10, ptr %p11, ptr %p12, ptr %p13, ptr %p14, ptr %p15, ptr %p16, ptr %p17, ptr %p18, ptr %p19, i64 %stride, i64 %extent, i64 %arg1, i1 %cond1, i1 %cond2, i64 %idx, <8 x i1> %mask1, i64 %offset, <8 x i1> %mask2) nounwind {
+; CHECK-LABEL: foo:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: pushq %rbp
+; CHECK-NEXT: pushq %r15
+; CHECK-NEXT: pushq %r14
+; CHECK-NEXT: pushq %r13
+; CHECK-NEXT: pushq %r12
+; CHECK-NEXT: pushq %rsi
+; CHECK-NEXT: pushq %rdi
+; CHECK-NEXT: pushq %rbx
+; CHECK-NEXT: subq $56, %rsp
+; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rbp
+; CHECK-NEXT: andq $-32, %rsp
+; CHECK-NEXT: movq %rsp, %rbx
+; CHECK-NEXT: movq 312(%rbp), %rax
+; CHECK-NEXT: vpmovsxwd (%rax), %ymm0
+; CHECK-NEXT: vpslld $31, %ymm0, %ymm0
+; CHECK-NEXT: vptestmd %ymm0, %ymm0, %k2
+; CHECK-NEXT: movq 304(%rbp), %r29
+; CHECK-NEXT: movzbl 296(%rbp), %r26d
+; CHECK-NEXT: movzbl 288(%rbp), %r30d
+; CHECK-NEXT: movq 280(%rbp), %r31
+; CHECK-NEXT: movq 272(%rbp), %r20
+; CHECK-NEXT: movq 264(%rbp), %r21
+; CHECK-NEXT: movq 240(%rbp), %r9
+; CHECK-NEXT: movq 232(%rbp), %r10
+; CHECK-NEXT: movq 224(%rbp), %r11
+; CHECK-NEXT: movq 216(%rbp), %r16
+; CHECK-NEXT: movq 208(%rbp), %r17
+; CHECK-NEXT: movq 192(%rbp), %r18
+; CHECK-NEXT: movq 184(%rbp), %r19
+; CHECK-NEXT: movq 176(%rbp), %r22
+; CHECK-NEXT: movq 168(%rbp), %r23
+; CHECK-NEXT: movq 160(%rbp), %r24
+; CHECK-NEXT: movq 152(%rbp), %r25
+; CHECK-NEXT: movq 144(%rbp), %r14
+; CHECK-NEXT: movq 136(%rbp), %r15
+; CHECK-NEXT: movq 128(%rbp), %r12
+; CHECK-NEXT: movq 120(%rbp), %r13
+; CHECK-NEXT: movq 112(%rbp), %rsi
+; CHECK-NEXT: leaq 15(,%r21,4), %rdx
+; CHECK-NEXT: andq $-16, %rdx
+; CHECK-NEXT: movq %rdx, %rax
+; CHECK-NEXT: callq ___chkstk_ms
+; CHECK-NEXT: subq %rax, %rsp
+; CHECK-NEXT: andq $-32, %rsp
+; CHECK-NEXT: movq %rdx, %rax
+; CHECK-NEXT: callq ___chkstk_ms
+; CHECK-NEXT: subq %rax, %rsp
+; CHECK-NEXT: andq $-32, %rsp
+; CHECK-NEXT: movl %ecx, %r8d
+; CHECK-NEXT: xorl %edi, %edi
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: kmovw %k2, 14(%rbx) # 2-byte Spill
+; CHECK-NEXT: movq %r8, 24(%rbx) # 8-byte Spill
+; CHECK-NEXT: jmp .LBB0_1
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB0_6: # %loop_outer.backedge
+; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT: subq %r21, %rdi
+; CHECK-NEXT: .LBB0_1: # %loop_outer
+; CHECK-NEXT: # =>This Loop Header: Depth=1
+; CHECK-NEXT: # Child Loop BB0_8 Depth 2
+; CHECK-NEXT: testq %r31, %r31
+; CHECK-NEXT: ctestgq {dfv=} %r20, %r20
+; CHECK-NEXT: jns .LBB0_10
+; CHECK-NEXT: # %bb.2: # %then_branch
+; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT: testb $1, %r30b
+; CHECK-NEXT: je .LBB0_3
+; CHECK-NEXT: # %bb.5: # %early_exit
+; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT: decq %rax
+; CHECK-NEXT: jmp .LBB0_6
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB0_10: # %else_branch
+; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT: leaq -1(%rax), %r27
+; CHECK-NEXT: testb $1, %r26b
+; CHECK-NEXT: movq %r27, 16(%rbx) # 8-byte Spill
+; CHECK-NEXT: je .LBB0_11
+; CHECK-NEXT: # %bb.7: # %memset_path
+; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT: subq $32, %rsp
+; CHECK-NEXT: xorl %ecx, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: callq memset
+; CHECK-NEXT: movzbl 288(%rbp), %r30d
+; CHECK-NEXT: addq $32, %rsp
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB0_8: # %inner_loop
+; CHECK-NEXT: # Parent Loop BB0_1 Depth=1
+; CHECK-NEXT: # => This Inner Loop Header: Depth=2
+; CHECK-NEXT: testb $1, %r30b
+; CHECK-NEXT: jne .LBB0_8
+; CHECK-NEXT: # %bb.9: # in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT: kmovw 14(%rbx), %k2 # 2-byte Reload
+; CHECK-NEXT: movq 304(%rbp), %r29
+; CHECK-NEXT: movq 240(%rbp), %r9
+; CHECK-NEXT: movq 232(%rbp), %r10
+; CHECK-NEXT: movq 224(%rbp), %r11
+; CHECK-NEXT: movq 216(%rbp), %r16
+; CHECK-NEXT: movq 208(%rbp), %r17
+; CHECK-NEXT: movq 192(%rbp), %r18
+; CHECK-NEXT: movq 184(%rbp), %r19
+; CHECK-NEXT: movq 176(%rbp), %r22
+; CHECK-NEXT: movq 168(%rbp), %r23
+; CHECK-NEXT: movq 160(%rbp), %r24
+; CHECK-NEXT: movq 152(%rbp), %r25
+; CHECK-NEXT: movzbl 296(%rbp), %r26d
+; CHECK-NEXT: movq 280(%rbp), %r31
+; CHECK-NEXT: movq 272(%rbp), %r20
+; CHECK-NEXT: movq 264(%rbp), %r21
+; CHECK-NEXT: movq 24(%rbx), %r8 # 8-byte Reload
+; CHECK-NEXT: movq 16(%rbx), %rax # 8-byte Reload
+; CHECK-NEXT: subq %r21, %rdi
+; CHECK-NEXT: jmp .LBB0_1
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB0_11: # %compute_path
+; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT: testb $1, %r30b
+; CHECK-NEXT: jne .LBB0_12
+; CHECK-NEXT: # %bb.14: # %loop_exit
+; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT: movq 256(%rbp), %rcx
+; CHECK-NEXT: addq %rax, %rcx
+; CHECK-NEXT: andl $1, %r21d, %r27d
+; CHECK-NEXT: andl $1, %r8d, %edx
+; CHECK-NEXT: movq %r20, %rax
+; CHECK-NEXT: movq %r31, %r20
+; CHECK-NEXT: movl %r26d, %r31d
+; CHECK-NEXT: movq 320(%rbp), %r26
+; CHECK-NEXT: movq 200(%rbp), %r28
+; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; CHECK-NEXT: vmovdqu32 %ymm0, (%r28,%r26) {%k2}
+; CHECK-NEXT: movl %r31d, %r26d
+; CHECK-NEXT: movq %r20, %r31
+; CHECK-NEXT: movq %rax, %r20
+; CHECK-NEXT: vmovups %ymm0, (%rcx) {%k2}
+; CHECK-NEXT: cmpl %edx, %r27d
+; CHECK-NEXT: movq 16(%rbx), %rax # 8-byte Reload
+; CHECK-NEXT: je .LBB0_6
+; CHECK-NEXT: # %bb.15:
+; CHECK-NEXT: movl $1, %eax
+; CHECK-NEXT: jmp .LBB0_13
+; CHECK-NEXT: .LBB0_3: # %vector_loop.preheader
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB0_4: # %vector_loop
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: vmovdqu (%r9), %ymm1
+; CHECK-NEXT: vpord (%r18), %ymm1, %ymm1
+; CHECK-NEXT: vmovups %ymm0, (%r23) {%k2}
+; CHECK-NEXT: vmovups %ymm0, (%r24) {%k2}
+; CHECK-NEXT: vmovups (%r25,%rdi), %ymm2
+; CHECK-NEXT: vcmpltps (%r14,%rdi), %ymm2, %k1
+; CHECK-NEXT: vptestnmd %ymm1, %ymm1, %k1 {%k1}
+; CHECK-NEXT: vmovups (%r10), %ymm1 {%k1} {z}
+; CHECK-NEXT: vmovups %ymm1, (%r22) {%k2}
+; CHECK-NEXT: vmovdqu32 %ymm0, (%r13) {%k2}
+; CHECK-NEXT: vmovdqu32 %ymm0, (%r16) {%k2}
+; CHECK-NEXT: vmovups %ymm0, (%r19) {%k2}
+; CHECK-NEXT: vmovups (%r15), %ymm1 {%k2} {z}
+; CHECK-NEXT: vmovups %ymm1, (%r17,%r29,4) {%k2}
+; CHECK-NEXT: vmovups %ymm0, (%r11) {%k2}
+; CHECK-NEXT: vmovups %ymm0, (%r12) {%k2}
+; CHECK-NEXT: vmovups %ymm0, (%rsi) {%k2}
+; CHECK-NEXT: incq %rax
+; CHECK-NEXT: addq $4, %rdi
+; CHECK-NEXT: jmp .LBB0_4
+; CHECK-NEXT: .LBB0_12:
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: .LBB0_13: # %exit
+; CHECK-NEXT: leaq 8(%rbp), %rsp
+; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: popq %rdi
+; CHECK-NEXT: popq %rsi
+; CHECK-NEXT: popq %r12
+; CHECK-NEXT: popq %r13
+; CHECK-NEXT: popq %r14
+; CHECK-NEXT: popq %r15
+; CHECK-NEXT: popq %rbp
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+entry:
+ %buf1 = alloca float, i64 %stride, align 32
+ %buf2 = alloca float, i64 %stride, align 32
+ %dim1_ext = zext i32 %dim1 to i64
+ br label %loop_outer
+
+loop_outer: ; preds = %loop_exit, %inner_loop, %early_exit, %entry
+ %iv = phi i64 [ 0, %entry ], [ %iv_dec, %early_exit ], [ %iv_dec2, %loop_exit ], [ %iv_dec2, %inner_loop ]
+ %cmp1 = icmp slt i64 0, %arg1
+ %cmp2 = icmp sgt i64 0, %extent
+ %both = and i1 %cmp1, %cmp2
+ br i1 %both, label %then_branch, label %else_branch
+
+then_branch: ; preds = %loop_outer
+ br i1 %cond1, label %early_exit, label %vector_loop
+
+early_exit: ; preds = %then_branch
+ %iv_dec = add i64 %iv, -1
+ br label %loop_outer
+
+else_branch: ; preds = %loop_outer
+ %iv_dec2 = add i64 %iv, -1
+ %gep1 = getelementptr i8, ptr %p19, i64 %iv
+ br i1 %cond2, label %memset_path, label %compute_path
+
+vector_loop: ; preds = %vector_loop, %then_branch
+ %inner_iv = phi i64 [ %inner_iv_next, %vector_loop ], [ 0, %then_branch ]
+ %gep2 = getelementptr i32, ptr %p10, i64 %inner_iv
+ %vec1 = load <8 x i32>, ptr %p17, align 1
+ %vec2 = load <8 x i32>, ptr %p11, align 1
+ %base_offset = mul i64 %stride, %iv
+ %base1 = getelementptr i8, ptr %p5, i64 %base_offset
+ %gep3 = getelementptr float, ptr %base1, i64 %inner_iv
+ %base2 = getelementptr i8, ptr %p6, i64 %base_offset
+ %gep4 = getelementptr float, ptr %base2, i64 %inner_iv
+ call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %p8, <8 x i1> %mask1)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %p7, <8 x i1> %mask1)
+ %or_vec = or <8 x i32> %vec1, %vec2
+ %cmp_vec1 = icmp eq <8 x i32> %or_vec, zeroinitializer
+ %load1 = load <8 x float>, ptr %gep3, align 1
+ %load2 = load <8 x float>, ptr %gep4, align 1
+ %cmp_vec2 = fcmp ogt <8 x float> %load1, %load2
+ %and_mask = and <8 x i1> %cmp_vec1, %cmp_vec2
+ %masked_load = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %p16, <8 x i1> %and_mask, <8 x float> zeroinitializer)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %masked_load, ptr %p9, <8 x i1> %mask1)
+ call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr %p2, <8 x i1> %mask1)
+ call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr %p14, <8 x i1> %mask1)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %p10, <8 x i1> %mask1)
+ %load3 = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %p4, <8 x i1> %mask1, <8 x float> zeroinitializer)
+ %gep5 = getelementptr float, ptr %p13, i64 %idx
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %load3, ptr %gep5, <8 x i1> %mask1)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %p15, <8 x i1> %mask1)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %p3, <8 x i1> %mask1)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %p1, <8 x i1> %mask1)
+ %inner_iv_next = add i64 %inner_iv, 1
+ br label %vector_loop
+
+memset_path: ; preds = %else_branch
+ call void @llvm.memset.p0.i64(ptr null, i8 0, i64 %dim1_ext, i1 false)
+ br label %inner_loop
+
+inner_loop: ; preds = %inner_loop, %memset_path
+ br i1 %cond1, label %inner_loop, label %loop_outer
+
+compute_path: ; preds = %else_branch
+ %and1 = and i64 %stride, 1
+ %and2 = and i64 %dim1_ext, 1
+ br i1 %cond1, label %exit, label %loop_exit
+
+exit: ; preds = %loop_exit, %compute_path
+ %ret_val = phi i64 [ 1, %loop_exit ], [ 0, %compute_path ]
+ ret i64 %ret_val
+
+loop_exit: ; preds = %compute_path
+ %gep6 = getelementptr i8, ptr %p12, i64 %offset
+ call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr %gep6, <8 x i1> %mask1)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %gep1, <8 x i1> %mask1)
+ %cmp_exit = icmp eq i64 %and1, %and2
+ br i1 %cmp_exit, label %loop_outer, label %exit
+}
More information about the llvm-commits
mailing list