[llvm] [RISCV] Match indices based on significant bits when forming strided ops (PR #65777)
Philip Reames via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 11 09:13:09 PDT 2023
https://github.com/preames updated https://github.com/llvm/llvm-project/pull/65777:
>From 67f1d6227ab62c7e289dedd45aa6fc00ec90da29 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Fri, 8 Sep 2023 08:36:55 -0700
Subject: [PATCH 1/2] [RISCV] Match indices based on significant bits when
forming strided ops
When checking to see if our index expressions can be converted into strided operations, we previously gave up if the index type wasn't an exact match for the intptrty for the address. Instead, we can ask known bits how many bits are needed, and proceed as long as the number of required bits fits in intptrty.
Note that the motivation of this is mostly test cleanup. We canonicalize at IR such that the gep index will match the intptrty. This is mostly useful so that we can write both RV32 and RV64 tests from the same source. Its also helpful in preventing confusion - I've stumbled across this at least four times now and wasted time each one.
Interestingly, this does appear to catch some loop cases that O3 does not canonicalize. I don't think the vectorizer is likely to emit indices narrower that intptr, but if another frontend did, this might pick them up.
This may also pick up some pass ordering problems, but that's a happy accident at best.
---
.../RISCV/RISCVGatherScatterLowering.cpp | 19 +-
.../RISCV/rvv/fixed-vectors-masked-gather.ll | 313 ++----------------
...xed-vectors-strided-load-store-negative.ll | 44 ---
.../rvv/fixed-vectors-strided-load-store.ll | 50 ++-
4 files changed, 87 insertions(+), 339 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
index 0e9244d0aefa813..30a5541fd77a258 100644
--- a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
@@ -373,14 +373,15 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr,
if (!VecOperand)
return std::make_pair(nullptr, nullptr);
- // We can't extract the stride if the arithmetic is done at a different size
- // than the pointer type. Adding the stride later may not wrap correctly.
- // Technically we could handle wider indices, but I don't expect that in
- // practice.
+ // We need the number of significant bits to match the index type. IF it
+ // doesn't, then adding the stride later may not wrap correctly.
Value *VecIndex = Ops[*VecOperand];
Type *VecIntPtrTy = DL->getIntPtrType(GEP->getType());
- if (VecIndex->getType() != VecIntPtrTy)
- return std::make_pair(nullptr, nullptr);
+ if (VecIndex->getType()->getScalarSizeInBits() > VecIntPtrTy->getScalarSizeInBits()) {
+ unsigned MaxBits = ComputeMaxSignificantBits(VecIndex, *DL);
+ if (MaxBits > VecIntPtrTy->getScalarSizeInBits())
+ return std::make_pair(nullptr, nullptr);
+ }
// Handle the non-recursive case. This is what we see if the vectorizer
// decides to use a scalar IV + vid on demand instead of a vector IV.
@@ -397,7 +398,8 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr,
// Convert stride to pointer size if needed.
Type *IntPtrTy = DL->getIntPtrType(BasePtr->getType());
- assert(Stride->getType() == IntPtrTy && "Unexpected type");
+ assert(IntPtrTy == VecIntPtrTy->getScalarType());
+ Stride = Builder.CreateSExtOrTrunc(Stride, IntPtrTy);
// Scale the stride by the size of the indexed type.
if (TypeScale != 1)
@@ -437,7 +439,8 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr,
// Convert stride to pointer size if needed.
Type *IntPtrTy = DL->getIntPtrType(BasePtr->getType());
- assert(Stride->getType() == IntPtrTy && "Unexpected type");
+ assert(IntPtrTy == VecIntPtrTy->getScalarType());
+ Stride = Builder.CreateSExtOrTrunc(Stride, IntPtrTy);
// Scale the stride by the size of the indexed type.
if (TypeScale != 1)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index f7352b4659e5a9b..ec5c076a04bc66b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -12857,54 +12857,14 @@ define <4 x i32> @mgather_broadcast_load_unmasked(ptr %base) {
;
; RV64V-LABEL: mgather_broadcast_load_unmasked:
; RV64V: # %bb.0:
-; RV64V-NEXT: vsetivli zero, 4, e64, m2, ta, ma
-; RV64V-NEXT: vmv.v.i v10, 0
-; RV64V-NEXT: vsetvli zero, zero, e32, m1, ta, ma
-; RV64V-NEXT: vluxei64.v v8, (a0), v10
+; RV64V-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV64V-NEXT: vlse32.v v8, (a0), zero
; RV64V-NEXT: ret
;
; RV64ZVE32F-LABEL: mgather_broadcast_load_unmasked:
; RV64ZVE32F: # %bb.0:
-; RV64ZVE32F-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; RV64ZVE32F-NEXT: vmset.m v8
-; RV64ZVE32F-NEXT: vmv.x.s a1, v8
-; RV64ZVE32F-NEXT: # implicit-def: $v8
-; RV64ZVE32F-NEXT: beqz zero, .LBB99_5
-; RV64ZVE32F-NEXT: # %bb.1: # %else
-; RV64ZVE32F-NEXT: andi a2, a1, 2
-; RV64ZVE32F-NEXT: bnez a2, .LBB99_6
-; RV64ZVE32F-NEXT: .LBB99_2: # %else2
-; RV64ZVE32F-NEXT: andi a2, a1, 4
-; RV64ZVE32F-NEXT: bnez a2, .LBB99_7
-; RV64ZVE32F-NEXT: .LBB99_3: # %else5
-; RV64ZVE32F-NEXT: andi a1, a1, 8
-; RV64ZVE32F-NEXT: bnez a1, .LBB99_8
-; RV64ZVE32F-NEXT: .LBB99_4: # %else8
-; RV64ZVE32F-NEXT: ret
-; RV64ZVE32F-NEXT: .LBB99_5: # %cond.load
; RV64ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; RV64ZVE32F-NEXT: vlse32.v v8, (a0), zero
-; RV64ZVE32F-NEXT: andi a2, a1, 2
-; RV64ZVE32F-NEXT: beqz a2, .LBB99_2
-; RV64ZVE32F-NEXT: .LBB99_6: # %cond.load1
-; RV64ZVE32F-NEXT: lw a2, 0(a0)
-; RV64ZVE32F-NEXT: vsetivli zero, 2, e32, m1, tu, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a2
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 1
-; RV64ZVE32F-NEXT: andi a2, a1, 4
-; RV64ZVE32F-NEXT: beqz a2, .LBB99_3
-; RV64ZVE32F-NEXT: .LBB99_7: # %cond.load4
-; RV64ZVE32F-NEXT: lw a2, 0(a0)
-; RV64ZVE32F-NEXT: vsetivli zero, 3, e32, m1, tu, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a2
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 2
-; RV64ZVE32F-NEXT: andi a1, a1, 8
-; RV64ZVE32F-NEXT: beqz a1, .LBB99_4
-; RV64ZVE32F-NEXT: .LBB99_8: # %cond.load7
-; RV64ZVE32F-NEXT: lw a0, 0(a0)
-; RV64ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a0
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 3
; RV64ZVE32F-NEXT: ret
%head = insertelement <4 x i1> poison, i1 true, i32 0
%allones = shufflevector <4 x i1> %head, <4 x i1> poison, <4 x i32> zeroinitializer
@@ -12949,54 +12909,14 @@ define <4 x i32> @mgather_broadcast_load_masked(ptr %base, <4 x i1> %m) {
;
; RV64V-LABEL: mgather_broadcast_load_masked:
; RV64V: # %bb.0:
-; RV64V-NEXT: vsetivli zero, 4, e64, m2, ta, ma
-; RV64V-NEXT: vmv.v.i v10, 0
-; RV64V-NEXT: vsetvli zero, zero, e32, m1, ta, ma
-; RV64V-NEXT: vluxei64.v v8, (a0), v10, v0.t
+; RV64V-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV64V-NEXT: vlse32.v v8, (a0), zero, v0.t
; RV64V-NEXT: ret
;
; RV64ZVE32F-LABEL: mgather_broadcast_load_masked:
; RV64ZVE32F: # %bb.0:
-; RV64ZVE32F-NEXT: vsetivli zero, 1, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT: vmv.x.s a1, v0
-; RV64ZVE32F-NEXT: andi a2, a1, 1
-; RV64ZVE32F-NEXT: # implicit-def: $v8
-; RV64ZVE32F-NEXT: bnez a2, .LBB101_5
-; RV64ZVE32F-NEXT: # %bb.1: # %else
-; RV64ZVE32F-NEXT: andi a2, a1, 2
-; RV64ZVE32F-NEXT: bnez a2, .LBB101_6
-; RV64ZVE32F-NEXT: .LBB101_2: # %else2
-; RV64ZVE32F-NEXT: andi a2, a1, 4
-; RV64ZVE32F-NEXT: bnez a2, .LBB101_7
-; RV64ZVE32F-NEXT: .LBB101_3: # %else5
-; RV64ZVE32F-NEXT: andi a1, a1, 8
-; RV64ZVE32F-NEXT: bnez a1, .LBB101_8
-; RV64ZVE32F-NEXT: .LBB101_4: # %else8
-; RV64ZVE32F-NEXT: ret
-; RV64ZVE32F-NEXT: .LBB101_5: # %cond.load
; RV64ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV64ZVE32F-NEXT: vlse32.v v8, (a0), zero
-; RV64ZVE32F-NEXT: andi a2, a1, 2
-; RV64ZVE32F-NEXT: beqz a2, .LBB101_2
-; RV64ZVE32F-NEXT: .LBB101_6: # %cond.load1
-; RV64ZVE32F-NEXT: lw a2, 0(a0)
-; RV64ZVE32F-NEXT: vsetivli zero, 2, e32, m1, tu, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a2
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 1
-; RV64ZVE32F-NEXT: andi a2, a1, 4
-; RV64ZVE32F-NEXT: beqz a2, .LBB101_3
-; RV64ZVE32F-NEXT: .LBB101_7: # %cond.load4
-; RV64ZVE32F-NEXT: lw a2, 0(a0)
-; RV64ZVE32F-NEXT: vsetivli zero, 3, e32, m1, tu, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a2
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 2
-; RV64ZVE32F-NEXT: andi a1, a1, 8
-; RV64ZVE32F-NEXT: beqz a1, .LBB101_4
-; RV64ZVE32F-NEXT: .LBB101_8: # %cond.load7
-; RV64ZVE32F-NEXT: lw a0, 0(a0)
-; RV64ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a0
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 3
+; RV64ZVE32F-NEXT: vlse32.v v8, (a0), zero, v0.t
; RV64ZVE32F-NEXT: ret
%head = insertelement <4 x i1> poison, i1 true, i32 0
%allones = shufflevector <4 x i1> %head, <4 x i1> poison, <4 x i32> zeroinitializer
@@ -13017,58 +12937,16 @@ define <4 x i32> @mgather_unit_stride_load(ptr %base) {
;
; RV64V-LABEL: mgather_unit_stride_load:
; RV64V: # %bb.0:
-; RV64V-NEXT: vsetivli zero, 4, e64, m2, ta, ma
-; RV64V-NEXT: vid.v v8
-; RV64V-NEXT: vsll.vi v10, v8, 2
-; RV64V-NEXT: vsetvli zero, zero, e32, m1, ta, ma
-; RV64V-NEXT: vluxei64.v v8, (a0), v10
+; RV64V-NEXT: li a1, 4
+; RV64V-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV64V-NEXT: vlse32.v v8, (a0), a1
; RV64V-NEXT: ret
;
; RV64ZVE32F-LABEL: mgather_unit_stride_load:
; RV64ZVE32F: # %bb.0:
-; RV64ZVE32F-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; RV64ZVE32F-NEXT: vmset.m v8
-; RV64ZVE32F-NEXT: vmv.x.s a1, v8
-; RV64ZVE32F-NEXT: # implicit-def: $v8
-; RV64ZVE32F-NEXT: beqz zero, .LBB102_5
-; RV64ZVE32F-NEXT: # %bb.1: # %else
-; RV64ZVE32F-NEXT: andi a2, a1, 2
-; RV64ZVE32F-NEXT: bnez a2, .LBB102_6
-; RV64ZVE32F-NEXT: .LBB102_2: # %else2
-; RV64ZVE32F-NEXT: andi a2, a1, 4
-; RV64ZVE32F-NEXT: bnez a2, .LBB102_7
-; RV64ZVE32F-NEXT: .LBB102_3: # %else5
-; RV64ZVE32F-NEXT: andi a1, a1, 8
-; RV64ZVE32F-NEXT: bnez a1, .LBB102_8
-; RV64ZVE32F-NEXT: .LBB102_4: # %else8
-; RV64ZVE32F-NEXT: ret
-; RV64ZVE32F-NEXT: .LBB102_5: # %cond.load
-; RV64ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV64ZVE32F-NEXT: vlse32.v v8, (a0), zero
-; RV64ZVE32F-NEXT: andi a2, a1, 2
-; RV64ZVE32F-NEXT: beqz a2, .LBB102_2
-; RV64ZVE32F-NEXT: .LBB102_6: # %cond.load1
-; RV64ZVE32F-NEXT: addi a2, a0, 4
-; RV64ZVE32F-NEXT: lw a2, 0(a2)
-; RV64ZVE32F-NEXT: vsetivli zero, 2, e32, m1, tu, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a2
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 1
-; RV64ZVE32F-NEXT: andi a2, a1, 4
-; RV64ZVE32F-NEXT: beqz a2, .LBB102_3
-; RV64ZVE32F-NEXT: .LBB102_7: # %cond.load4
-; RV64ZVE32F-NEXT: addi a2, a0, 8
-; RV64ZVE32F-NEXT: lw a2, 0(a2)
-; RV64ZVE32F-NEXT: vsetivli zero, 3, e32, m1, tu, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a2
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 2
-; RV64ZVE32F-NEXT: andi a1, a1, 8
-; RV64ZVE32F-NEXT: beqz a1, .LBB102_4
-; RV64ZVE32F-NEXT: .LBB102_8: # %cond.load7
-; RV64ZVE32F-NEXT: addi a0, a0, 12
-; RV64ZVE32F-NEXT: lw a0, 0(a0)
+; RV64ZVE32F-NEXT: li a1, 4
; RV64ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a0
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 3
+; RV64ZVE32F-NEXT: vlse32.v v8, (a0), a1
; RV64ZVE32F-NEXT: ret
%head = insertelement <4 x i1> poison, i1 true, i32 0
%allones = shufflevector <4 x i1> %head, <4 x i1> poison, <4 x i32> zeroinitializer
@@ -13089,61 +12967,18 @@ define <4 x i32> @mgather_unit_stride_load_with_offset(ptr %base) {
;
; RV64V-LABEL: mgather_unit_stride_load_with_offset:
; RV64V: # %bb.0:
-; RV64V-NEXT: lui a1, 115073
-; RV64V-NEXT: addiw a1, a1, 1040
-; RV64V-NEXT: vsetivli zero, 4, e64, m2, ta, ma
-; RV64V-NEXT: vmv.s.x v8, a1
-; RV64V-NEXT: vsext.vf8 v10, v8
-; RV64V-NEXT: vsetvli zero, zero, e32, m1, ta, ma
-; RV64V-NEXT: vluxei64.v v8, (a0), v10
+; RV64V-NEXT: addi a0, a0, 16
+; RV64V-NEXT: li a1, 4
+; RV64V-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV64V-NEXT: vlse32.v v8, (a0), a1
; RV64V-NEXT: ret
;
; RV64ZVE32F-LABEL: mgather_unit_stride_load_with_offset:
; RV64ZVE32F: # %bb.0:
-; RV64ZVE32F-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; RV64ZVE32F-NEXT: vmset.m v8
-; RV64ZVE32F-NEXT: vmv.x.s a1, v8
-; RV64ZVE32F-NEXT: # implicit-def: $v8
-; RV64ZVE32F-NEXT: beqz zero, .LBB103_5
-; RV64ZVE32F-NEXT: # %bb.1: # %else
-; RV64ZVE32F-NEXT: andi a2, a1, 2
-; RV64ZVE32F-NEXT: bnez a2, .LBB103_6
-; RV64ZVE32F-NEXT: .LBB103_2: # %else2
-; RV64ZVE32F-NEXT: andi a2, a1, 4
-; RV64ZVE32F-NEXT: bnez a2, .LBB103_7
-; RV64ZVE32F-NEXT: .LBB103_3: # %else5
-; RV64ZVE32F-NEXT: andi a1, a1, 8
-; RV64ZVE32F-NEXT: bnez a1, .LBB103_8
-; RV64ZVE32F-NEXT: .LBB103_4: # %else8
-; RV64ZVE32F-NEXT: ret
-; RV64ZVE32F-NEXT: .LBB103_5: # %cond.load
-; RV64ZVE32F-NEXT: addi a2, a0, 16
+; RV64ZVE32F-NEXT: addi a0, a0, 16
+; RV64ZVE32F-NEXT: li a1, 4
; RV64ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV64ZVE32F-NEXT: vlse32.v v8, (a2), zero
-; RV64ZVE32F-NEXT: andi a2, a1, 2
-; RV64ZVE32F-NEXT: beqz a2, .LBB103_2
-; RV64ZVE32F-NEXT: .LBB103_6: # %cond.load1
-; RV64ZVE32F-NEXT: addi a2, a0, 20
-; RV64ZVE32F-NEXT: lw a2, 0(a2)
-; RV64ZVE32F-NEXT: vsetivli zero, 2, e32, m1, tu, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a2
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 1
-; RV64ZVE32F-NEXT: andi a2, a1, 4
-; RV64ZVE32F-NEXT: beqz a2, .LBB103_3
-; RV64ZVE32F-NEXT: .LBB103_7: # %cond.load4
-; RV64ZVE32F-NEXT: addi a2, a0, 24
-; RV64ZVE32F-NEXT: lw a2, 0(a2)
-; RV64ZVE32F-NEXT: vsetivli zero, 3, e32, m1, tu, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a2
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 2
-; RV64ZVE32F-NEXT: andi a1, a1, 8
-; RV64ZVE32F-NEXT: beqz a1, .LBB103_4
-; RV64ZVE32F-NEXT: .LBB103_8: # %cond.load7
-; RV64ZVE32F-NEXT: addi a0, a0, 28
-; RV64ZVE32F-NEXT: lw a0, 0(a0)
-; RV64ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a0
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 3
+; RV64ZVE32F-NEXT: vlse32.v v8, (a0), a1
; RV64ZVE32F-NEXT: ret
%head = insertelement <4 x i1> poison, i1 true, i32 0
%allones = shufflevector <4 x i1> %head, <4 x i1> poison, <4 x i32> zeroinitializer
@@ -13155,66 +12990,23 @@ define <4 x i32> @mgather_unit_stride_load_with_offset(ptr %base) {
define <4 x i32> @mgather_unit_stride_load_narrow_idx(ptr %base) {
; RV32-LABEL: mgather_unit_stride_load_narrow_idx:
; RV32: # %bb.0:
+; RV32-NEXT: li a1, 4
; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT: vid.v v8
-; RV32-NEXT: vsll.vi v8, v8, 2
-; RV32-NEXT: vluxei32.v v8, (a0), v8
+; RV32-NEXT: vlse32.v v8, (a0), a1
; RV32-NEXT: ret
;
; RV64V-LABEL: mgather_unit_stride_load_narrow_idx:
; RV64V: # %bb.0:
-; RV64V-NEXT: vsetivli zero, 4, e64, m2, ta, ma
-; RV64V-NEXT: vid.v v8
-; RV64V-NEXT: vsll.vi v10, v8, 2
-; RV64V-NEXT: vsetvli zero, zero, e32, m1, ta, ma
-; RV64V-NEXT: vluxei64.v v8, (a0), v10
+; RV64V-NEXT: li a1, 4
+; RV64V-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV64V-NEXT: vlse32.v v8, (a0), a1
; RV64V-NEXT: ret
;
; RV64ZVE32F-LABEL: mgather_unit_stride_load_narrow_idx:
; RV64ZVE32F: # %bb.0:
-; RV64ZVE32F-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; RV64ZVE32F-NEXT: vmset.m v8
-; RV64ZVE32F-NEXT: vmv.x.s a1, v8
-; RV64ZVE32F-NEXT: # implicit-def: $v8
-; RV64ZVE32F-NEXT: beqz zero, .LBB104_5
-; RV64ZVE32F-NEXT: # %bb.1: # %else
-; RV64ZVE32F-NEXT: andi a2, a1, 2
-; RV64ZVE32F-NEXT: bnez a2, .LBB104_6
-; RV64ZVE32F-NEXT: .LBB104_2: # %else2
-; RV64ZVE32F-NEXT: andi a2, a1, 4
-; RV64ZVE32F-NEXT: bnez a2, .LBB104_7
-; RV64ZVE32F-NEXT: .LBB104_3: # %else5
-; RV64ZVE32F-NEXT: andi a1, a1, 8
-; RV64ZVE32F-NEXT: bnez a1, .LBB104_8
-; RV64ZVE32F-NEXT: .LBB104_4: # %else8
-; RV64ZVE32F-NEXT: ret
-; RV64ZVE32F-NEXT: .LBB104_5: # %cond.load
-; RV64ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV64ZVE32F-NEXT: vlse32.v v8, (a0), zero
-; RV64ZVE32F-NEXT: andi a2, a1, 2
-; RV64ZVE32F-NEXT: beqz a2, .LBB104_2
-; RV64ZVE32F-NEXT: .LBB104_6: # %cond.load1
-; RV64ZVE32F-NEXT: addi a2, a0, 4
-; RV64ZVE32F-NEXT: lw a2, 0(a2)
-; RV64ZVE32F-NEXT: vsetivli zero, 2, e32, m1, tu, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a2
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 1
-; RV64ZVE32F-NEXT: andi a2, a1, 4
-; RV64ZVE32F-NEXT: beqz a2, .LBB104_3
-; RV64ZVE32F-NEXT: .LBB104_7: # %cond.load4
-; RV64ZVE32F-NEXT: addi a2, a0, 8
-; RV64ZVE32F-NEXT: lw a2, 0(a2)
-; RV64ZVE32F-NEXT: vsetivli zero, 3, e32, m1, tu, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a2
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 2
-; RV64ZVE32F-NEXT: andi a1, a1, 8
-; RV64ZVE32F-NEXT: beqz a1, .LBB104_4
-; RV64ZVE32F-NEXT: .LBB104_8: # %cond.load7
-; RV64ZVE32F-NEXT: addi a0, a0, 12
-; RV64ZVE32F-NEXT: lw a0, 0(a0)
+; RV64ZVE32F-NEXT: li a1, 4
; RV64ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a0
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 3
+; RV64ZVE32F-NEXT: vlse32.v v8, (a0), a1
; RV64ZVE32F-NEXT: ret
%head = insertelement <4 x i1> poison, i1 true, i32 0
%allones = shufflevector <4 x i1> %head, <4 x i1> poison, <4 x i32> zeroinitializer
@@ -13226,66 +13018,23 @@ define <4 x i32> @mgather_unit_stride_load_narrow_idx(ptr %base) {
define <4 x i32> @mgather_unit_stride_load_wide_idx(ptr %base) {
; RV32-LABEL: mgather_unit_stride_load_wide_idx:
; RV32: # %bb.0:
+; RV32-NEXT: li a1, 4
; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT: vid.v v8
-; RV32-NEXT: vsll.vi v8, v8, 2
-; RV32-NEXT: vluxei32.v v8, (a0), v8
+; RV32-NEXT: vlse32.v v8, (a0), a1
; RV32-NEXT: ret
;
; RV64V-LABEL: mgather_unit_stride_load_wide_idx:
; RV64V: # %bb.0:
-; RV64V-NEXT: vsetivli zero, 4, e64, m2, ta, ma
-; RV64V-NEXT: vid.v v8
-; RV64V-NEXT: vsll.vi v10, v8, 2
-; RV64V-NEXT: vsetvli zero, zero, e32, m1, ta, ma
-; RV64V-NEXT: vluxei64.v v8, (a0), v10
+; RV64V-NEXT: li a1, 4
+; RV64V-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV64V-NEXT: vlse32.v v8, (a0), a1
; RV64V-NEXT: ret
;
; RV64ZVE32F-LABEL: mgather_unit_stride_load_wide_idx:
; RV64ZVE32F: # %bb.0:
-; RV64ZVE32F-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; RV64ZVE32F-NEXT: vmset.m v8
-; RV64ZVE32F-NEXT: vmv.x.s a1, v8
-; RV64ZVE32F-NEXT: # implicit-def: $v8
-; RV64ZVE32F-NEXT: beqz zero, .LBB105_5
-; RV64ZVE32F-NEXT: # %bb.1: # %else
-; RV64ZVE32F-NEXT: andi a2, a1, 2
-; RV64ZVE32F-NEXT: bnez a2, .LBB105_6
-; RV64ZVE32F-NEXT: .LBB105_2: # %else2
-; RV64ZVE32F-NEXT: andi a2, a1, 4
-; RV64ZVE32F-NEXT: bnez a2, .LBB105_7
-; RV64ZVE32F-NEXT: .LBB105_3: # %else5
-; RV64ZVE32F-NEXT: andi a1, a1, 8
-; RV64ZVE32F-NEXT: bnez a1, .LBB105_8
-; RV64ZVE32F-NEXT: .LBB105_4: # %else8
-; RV64ZVE32F-NEXT: ret
-; RV64ZVE32F-NEXT: .LBB105_5: # %cond.load
+; RV64ZVE32F-NEXT: li a1, 4
; RV64ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV64ZVE32F-NEXT: vlse32.v v8, (a0), zero
-; RV64ZVE32F-NEXT: andi a2, a1, 2
-; RV64ZVE32F-NEXT: beqz a2, .LBB105_2
-; RV64ZVE32F-NEXT: .LBB105_6: # %cond.load1
-; RV64ZVE32F-NEXT: addi a2, a0, 4
-; RV64ZVE32F-NEXT: lw a2, 0(a2)
-; RV64ZVE32F-NEXT: vsetivli zero, 2, e32, m1, tu, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a2
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 1
-; RV64ZVE32F-NEXT: andi a2, a1, 4
-; RV64ZVE32F-NEXT: beqz a2, .LBB105_3
-; RV64ZVE32F-NEXT: .LBB105_7: # %cond.load4
-; RV64ZVE32F-NEXT: addi a2, a0, 8
-; RV64ZVE32F-NEXT: lw a2, 0(a2)
-; RV64ZVE32F-NEXT: vsetivli zero, 3, e32, m1, tu, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a2
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 2
-; RV64ZVE32F-NEXT: andi a1, a1, 8
-; RV64ZVE32F-NEXT: beqz a1, .LBB105_4
-; RV64ZVE32F-NEXT: .LBB105_8: # %cond.load7
-; RV64ZVE32F-NEXT: addi a0, a0, 12
-; RV64ZVE32F-NEXT: lw a0, 0(a0)
-; RV64ZVE32F-NEXT: vsetivli zero, 4, e32, m1, ta, ma
-; RV64ZVE32F-NEXT: vmv.s.x v9, a0
-; RV64ZVE32F-NEXT: vslideup.vi v8, v9, 3
+; RV64ZVE32F-NEXT: vlse32.v v8, (a0), a1
; RV64ZVE32F-NEXT: ret
%head = insertelement <4 x i1> poison, i1 true, i32 0
%allones = shufflevector <4 x i1> %head, <4 x i1> poison, <4 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-negative.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-negative.ll
index 32aec9586c29cb2..f27002c6e78d8be 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-negative.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-negative.ll
@@ -50,50 +50,6 @@ for.cond.cleanup: ; preds = %vector.body
ret void
}
-; Don't transform since we might not handle wrap correctly with narrow indices.
-define void @gather_narrow_index(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {
-; CHECK-LABEL: @gather_narrow_index(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <32 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = mul nuw nsw <32 x i32> [[VEC_IND]], <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], <32 x i32> [[TMP0]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> [[TMP1]], i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP2]], align 1
-; CHECK-NEXT: [[TMP4:%.*]] = add <32 x i8> [[WIDE_LOAD]], [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT: store <32 x i8> [[TMP4]], ptr [[TMP2]], align 1
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <32 x i32> [[VEC_IND]], <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP6]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %vector.body
-
-vector.body: ; preds = %vector.body, %entry
- %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
- %vec.ind = phi <32 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>, %entry ], [ %vec.ind.next, %vector.body ]
- %0 = mul nuw nsw <32 x i32> %vec.ind, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
- %1 = getelementptr inbounds i8, ptr %B, <32 x i32> %0
- %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %1, i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
- %2 = getelementptr inbounds i8, ptr %A, i64 %index
- %wide.load = load <32 x i8>, ptr %2, align 1
- %3 = add <32 x i8> %wide.load, %wide.masked.gather
- store <32 x i8> %3, ptr %2, align 1
- %index.next = add nuw i64 %index, 32
- %vec.ind.next = add <32 x i32> %vec.ind, <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>
- %4 = icmp eq i64 %index.next, 1024
- br i1 %4, label %for.cond.cleanup, label %vector.body
-
-for.cond.cleanup: ; preds = %vector.body
- ret void
-}
-
; The last element of the start value of the phi has the wrong stride.
define void @gather_broken_stride(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {
; CHECK-LABEL: @gather_broken_stride(
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store.ll
index b359f71be0e67f5..838453b593c4302 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store.ll
@@ -970,16 +970,15 @@ define void @gather_narrow_idx(ptr noalias nocapture %A, ptr noalias nocapture r
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <32 x i16> [ <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 20, i16 21, i16 22, i16 23, i16 24, i16 25, i16 26, i16 27, i16 28, i16 29, i16 30, i16 31>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[I:%.*]] = mul nuw nsw <32 x i16> [[VEC_IND]], <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
-; CHECK-NEXT: [[I1:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], <32 x i16> [[I]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> [[I1]], i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
+; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i16 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[B:%.*]], i16 [[VEC_IND_SCALAR]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <32 x i8> @llvm.riscv.masked.strided.load.v32i8.p0.i64(<32 x i8> undef, ptr [[TMP0]], i64 5, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
; CHECK-NEXT: [[I2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[I2]], align 1
; CHECK-NEXT: [[I4:%.*]] = add <32 x i8> [[WIDE_LOAD]], [[WIDE_MASKED_GATHER]]
; CHECK-NEXT: store <32 x i8> [[I4]], ptr [[I2]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <32 x i16> [[VEC_IND]], <i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32>
+; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i16 [[VEC_IND_SCALAR]], 160
; CHECK-NEXT: [[I6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-NEXT: br i1 [[I6]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]
; CHECK: for.cond.cleanup:
@@ -1007,3 +1006,44 @@ for.cond.cleanup: ; preds = %vector.body
ret void
}
+define void @gather_narrow_index2(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {
+; CHECK-LABEL: @gather_narrow_index2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 [[VEC_IND_SCALAR]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <32 x i8> @llvm.riscv.masked.strided.load.v32i8.p0.i64(<32 x i8> undef, ptr [[TMP0]], i64 5, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = add <32 x i8> [[WIDE_LOAD]], [[WIDE_MASKED_GATHER]]
+; CHECK-NEXT: store <32 x i8> [[TMP2]], ptr [[TMP1]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i32 [[VEC_IND_SCALAR]], 160
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]
+; CHECK: for.cond.cleanup:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %entry
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.ind = phi <32 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>, %entry ], [ %vec.ind.next, %vector.body ]
+ %0 = mul nuw nsw <32 x i32> %vec.ind, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
+ %1 = getelementptr inbounds i8, ptr %B, <32 x i32> %0
+ %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %1, i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
+ %2 = getelementptr inbounds i8, ptr %A, i64 %index
+ %wide.load = load <32 x i8>, ptr %2, align 1
+ %3 = add <32 x i8> %wide.load, %wide.masked.gather
+ store <32 x i8> %3, ptr %2, align 1
+ %index.next = add nuw i64 %index, 32
+ %vec.ind.next = add <32 x i32> %vec.ind, <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>
+ %4 = icmp eq i64 %index.next, 1024
+ br i1 %4, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup: ; preds = %vector.body
+ ret void
+}
>From c03ef2629afcfa9fbd209cd2de223b1a4b75b78b Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Mon, 11 Sep 2023 09:11:47 -0700
Subject: [PATCH 2/2] Restrict to constants to avoid indexing problems
---
.../RISCV/RISCVGatherScatterLowering.cpp | 23 +++++----
...xed-vectors-strided-load-store-negative.ll | 44 ++++++++++++++++
.../rvv/fixed-vectors-strided-load-store.ll | 50 ++-----------------
3 files changed, 63 insertions(+), 54 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
index 30a5541fd77a258..5ad1e082344e77a 100644
--- a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
@@ -373,14 +373,21 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr,
if (!VecOperand)
return std::make_pair(nullptr, nullptr);
- // We need the number of significant bits to match the index type. IF it
- // doesn't, then adding the stride later may not wrap correctly.
+ // We can't extract the stride if the arithmetic is done at a different size
+ // than the pointer type. Adding the stride later may not wrap correctly.
+ // Technically we could handle wider indices, but I don't expect that in
+ // practice. Handle one special case here - constants. This simplifies
+ // writing test cases.
Value *VecIndex = Ops[*VecOperand];
Type *VecIntPtrTy = DL->getIntPtrType(GEP->getType());
- if (VecIndex->getType()->getScalarSizeInBits() > VecIntPtrTy->getScalarSizeInBits()) {
- unsigned MaxBits = ComputeMaxSignificantBits(VecIndex, *DL);
- if (MaxBits > VecIntPtrTy->getScalarSizeInBits())
+ if (VecIndex->getType() != VecIntPtrTy) {
+ auto *VecIndexC = dyn_cast<Constant>(VecIndex);
+ if (!VecIndexC)
return std::make_pair(nullptr, nullptr);
+ if (VecIndex->getType()->getScalarSizeInBits() > VecIntPtrTy->getScalarSizeInBits())
+ VecIndex = ConstantFoldCastInstruction(Instruction::Trunc, VecIndexC, VecIntPtrTy);
+ else
+ VecIndex = ConstantFoldCastInstruction(Instruction::SExt, VecIndexC, VecIntPtrTy);
}
// Handle the non-recursive case. This is what we see if the vectorizer
@@ -398,8 +405,7 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr,
// Convert stride to pointer size if needed.
Type *IntPtrTy = DL->getIntPtrType(BasePtr->getType());
- assert(IntPtrTy == VecIntPtrTy->getScalarType());
- Stride = Builder.CreateSExtOrTrunc(Stride, IntPtrTy);
+ assert(Stride->getType() == IntPtrTy && "Unexpected type");
// Scale the stride by the size of the indexed type.
if (TypeScale != 1)
@@ -439,8 +445,7 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr,
// Convert stride to pointer size if needed.
Type *IntPtrTy = DL->getIntPtrType(BasePtr->getType());
- assert(IntPtrTy == VecIntPtrTy->getScalarType());
- Stride = Builder.CreateSExtOrTrunc(Stride, IntPtrTy);
+ assert(Stride->getType() == IntPtrTy && "Unexpected type");
// Scale the stride by the size of the indexed type.
if (TypeScale != 1)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-negative.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-negative.ll
index f27002c6e78d8be..32aec9586c29cb2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-negative.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-negative.ll
@@ -50,6 +50,50 @@ for.cond.cleanup: ; preds = %vector.body
ret void
}
+; Don't transform since we might not handle wrap correctly with narrow indices.
+define void @gather_narrow_index(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {
+; CHECK-LABEL: @gather_narrow_index(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <32 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = mul nuw nsw <32 x i32> [[VEC_IND]], <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], <32 x i32> [[TMP0]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> [[TMP1]], i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = add <32 x i8> [[WIDE_LOAD]], [[WIDE_MASKED_GATHER]]
+; CHECK-NEXT: store <32 x i8> [[TMP4]], ptr [[TMP2]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <32 x i32> [[VEC_IND]], <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP6]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]
+; CHECK: for.cond.cleanup:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %entry
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.ind = phi <32 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>, %entry ], [ %vec.ind.next, %vector.body ]
+ %0 = mul nuw nsw <32 x i32> %vec.ind, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
+ %1 = getelementptr inbounds i8, ptr %B, <32 x i32> %0
+ %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %1, i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
+ %2 = getelementptr inbounds i8, ptr %A, i64 %index
+ %wide.load = load <32 x i8>, ptr %2, align 1
+ %3 = add <32 x i8> %wide.load, %wide.masked.gather
+ store <32 x i8> %3, ptr %2, align 1
+ %index.next = add nuw i64 %index, 32
+ %vec.ind.next = add <32 x i32> %vec.ind, <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>
+ %4 = icmp eq i64 %index.next, 1024
+ br i1 %4, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup: ; preds = %vector.body
+ ret void
+}
+
; The last element of the start value of the phi has the wrong stride.
define void @gather_broken_stride(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {
; CHECK-LABEL: @gather_broken_stride(
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store.ll
index 838453b593c4302..b359f71be0e67f5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store.ll
@@ -970,15 +970,16 @@ define void @gather_narrow_idx(ptr noalias nocapture %A, ptr noalias nocapture r
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i16 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[B:%.*]], i16 [[VEC_IND_SCALAR]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <32 x i8> @llvm.riscv.masked.strided.load.v32i8.p0.i64(<32 x i8> undef, ptr [[TMP0]], i64 5, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <32 x i16> [ <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 20, i16 21, i16 22, i16 23, i16 24, i16 25, i16 26, i16 27, i16 28, i16 29, i16 30, i16 31>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[I:%.*]] = mul nuw nsw <32 x i16> [[VEC_IND]], <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+; CHECK-NEXT: [[I1:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], <32 x i16> [[I]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> [[I1]], i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
; CHECK-NEXT: [[I2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[I2]], align 1
; CHECK-NEXT: [[I4:%.*]] = add <32 x i8> [[WIDE_LOAD]], [[WIDE_MASKED_GATHER]]
; CHECK-NEXT: store <32 x i8> [[I4]], ptr [[I2]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i16 [[VEC_IND_SCALAR]], 160
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <32 x i16> [[VEC_IND]], <i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32, i16 32>
; CHECK-NEXT: [[I6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-NEXT: br i1 [[I6]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]
; CHECK: for.cond.cleanup:
@@ -1006,44 +1007,3 @@ for.cond.cleanup: ; preds = %vector.body
ret void
}
-define void @gather_narrow_index2(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {
-; CHECK-LABEL: @gather_narrow_index2(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND_SCALAR:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[VEC_IND_NEXT_SCALAR:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[B:%.*]], i32 [[VEC_IND_SCALAR]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <32 x i8> @llvm.riscv.masked.strided.load.v32i8.p0.i64(<32 x i8> undef, ptr [[TMP0]], i64 5, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP1]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = add <32 x i8> [[WIDE_LOAD]], [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT: store <32 x i8> [[TMP2]], ptr [[TMP1]], align 1
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-NEXT: [[VEC_IND_NEXT_SCALAR]] = add i32 [[VEC_IND_SCALAR]], 160
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %vector.body
-
-vector.body: ; preds = %vector.body, %entry
- %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
- %vec.ind = phi <32 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>, %entry ], [ %vec.ind.next, %vector.body ]
- %0 = mul nuw nsw <32 x i32> %vec.ind, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
- %1 = getelementptr inbounds i8, ptr %B, <32 x i32> %0
- %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %1, i32 1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> undef)
- %2 = getelementptr inbounds i8, ptr %A, i64 %index
- %wide.load = load <32 x i8>, ptr %2, align 1
- %3 = add <32 x i8> %wide.load, %wide.masked.gather
- store <32 x i8> %3, ptr %2, align 1
- %index.next = add nuw i64 %index, 32
- %vec.ind.next = add <32 x i32> %vec.ind, <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>
- %4 = icmp eq i64 %index.next, 1024
- br i1 %4, label %for.cond.cleanup, label %vector.body
-
-for.cond.cleanup: ; preds = %vector.body
- ret void
-}
More information about the llvm-commits
mailing list