[llvm] [AMDGPU] Cost of i8 vector insert/extract is free in some cases (PR #194991)
Janek van Oirschot via llvm-commits
llvm-commits at lists.llvm.org
Tue May 5 07:32:30 PDT 2026
================
@@ -0,0 +1,253 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -passes=vector-combine < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -passes=vector-combine < %s | FileCheck -check-prefix=GFX12 %s
+
+; Test that VectorCombine does not scalarize v16i8/v8i8/v4i8 loads from LDS.
+; Vector loads can be efficiently lowered to ds_read_b128/b64/b32 instructions,
+; so scalarization would be a pessimization.
+
+ at lds = external addrspace(3) global [0 x i8], align 16
+
+define void @test_v16i8_load_with_extracts_no_scalarize(ptr addrspace(1) %out) {
+; GFX9-LABEL: @test_v16i8_load_with_extracts_no_scalarize(
+; GFX9-NEXT: [[PTR:%.*]] = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 0
+; GFX9-NEXT: [[VAL:%.*]] = load <16 x i8>, ptr addrspace(3) [[PTR]], align 16
+; GFX9-NEXT: [[E0:%.*]] = extractelement <16 x i8> [[VAL]], i64 0
+; GFX9-NEXT: [[E1:%.*]] = extractelement <16 x i8> [[VAL]], i64 1
+; GFX9-NEXT: [[E3:%.*]] = extractelement <16 x i8> [[VAL]], i64 3
+; GFX9-NEXT: [[E12:%.*]] = extractelement <16 x i8> [[VAL]], i64 8
+; GFX9-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT:%.*]], i64 0
+; GFX9-NEXT: store i8 [[E0]], ptr addrspace(1) [[P0]], align 1
+; GFX9-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 4
+; GFX9-NEXT: store i8 [[E1]], ptr addrspace(1) [[P4]], align 1
+; GFX9-NEXT: [[P8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 8
+; GFX9-NEXT: store i8 [[E3]], ptr addrspace(1) [[P8]], align 1
+; GFX9-NEXT: [[P12:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 11
+; GFX9-NEXT: store i8 [[E12]], ptr addrspace(1) [[P12]], align 1
+; GFX9-NEXT: ret void
+;
+; GFX12-LABEL: @test_v16i8_load_with_extracts_no_scalarize(
+; GFX12-NEXT: [[PTR:%.*]] = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 0
+; GFX12-NEXT: [[VAL:%.*]] = load <16 x i8>, ptr addrspace(3) [[PTR]], align 16
+; GFX12-NEXT: [[E0:%.*]] = extractelement <16 x i8> [[VAL]], i64 0
+; GFX12-NEXT: [[E1:%.*]] = extractelement <16 x i8> [[VAL]], i64 1
+; GFX12-NEXT: [[E3:%.*]] = extractelement <16 x i8> [[VAL]], i64 3
+; GFX12-NEXT: [[E12:%.*]] = extractelement <16 x i8> [[VAL]], i64 8
+; GFX12-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT:%.*]], i64 0
+; GFX12-NEXT: store i8 [[E0]], ptr addrspace(1) [[P0]], align 1
+; GFX12-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 4
+; GFX12-NEXT: store i8 [[E1]], ptr addrspace(1) [[P4]], align 1
+; GFX12-NEXT: [[P8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 8
+; GFX12-NEXT: store i8 [[E3]], ptr addrspace(1) [[P8]], align 1
+; GFX12-NEXT: [[P12:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 11
+; GFX12-NEXT: store i8 [[E12]], ptr addrspace(1) [[P12]], align 1
+; GFX12-NEXT: ret void
+;
+ %ptr = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 0
+ %val = load <16 x i8>, ptr addrspace(3) %ptr, align 16
+ %e0 = extractelement <16 x i8> %val, i64 0
+ %e1 = extractelement <16 x i8> %val, i64 1
+ %e3 = extractelement <16 x i8> %val, i64 3
+ %e8 = extractelement <16 x i8> %val, i64 8
+ %p0 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 0
+ store i8 %e0, ptr addrspace(1) %p0, align 1
+ %p4 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 4
+ store i8 %e1, ptr addrspace(1) %p4, align 1
+ %p8 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 8
+ store i8 %e3, ptr addrspace(1) %p8, align 1
+ %p11 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 11
+ store i8 %e8, ptr addrspace(1) %p11, align 1
+ ret void
+}
+
+; VectorCombine will scalarize when the cost model assigns a cost to
+; extract elements, which occurs when the index values are no aligned.
+define void @test_v16i8_load_with_extracts_scalarize(ptr addrspace(1) %out) {
+; GFX9-LABEL: @test_v16i8_load_with_extracts_scalarize(
+; GFX9-NEXT: [[PTR:%.*]] = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 0
+; GFX9-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr addrspace(3) [[PTR]], i32 0, i64 1
+; GFX9-NEXT: [[E1:%.*]] = load i8, ptr addrspace(3) [[TMP1]], align 1
+; GFX9-NEXT: [[TMP2:%.*]] = getelementptr inbounds <16 x i8>, ptr addrspace(3) [[PTR]], i32 0, i64 3
+; GFX9-NEXT: [[E3:%.*]] = load i8, ptr addrspace(3) [[TMP2]], align 1
+; GFX9-NEXT: [[TMP3:%.*]] = getelementptr inbounds <16 x i8>, ptr addrspace(3) [[PTR]], i32 0, i64 5
+; GFX9-NEXT: [[E5:%.*]] = load i8, ptr addrspace(3) [[TMP3]], align 1
+; GFX9-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT:%.*]], i64 3
+; GFX9-NEXT: store i8 [[E1]], ptr addrspace(1) [[P4]], align 1
+; GFX9-NEXT: [[P8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 8
+; GFX9-NEXT: store i8 [[E3]], ptr addrspace(1) [[P8]], align 1
+; GFX9-NEXT: [[P11:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 11
+; GFX9-NEXT: store i8 [[E5]], ptr addrspace(1) [[P11]], align 1
+; GFX9-NEXT: ret void
+;
+; GFX12-LABEL: @test_v16i8_load_with_extracts_scalarize(
+; GFX12-NEXT: [[PTR:%.*]] = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 0
+; GFX12-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr addrspace(3) [[PTR]], i32 0, i64 1
+; GFX12-NEXT: [[E1:%.*]] = load i8, ptr addrspace(3) [[TMP1]], align 1
+; GFX12-NEXT: [[TMP2:%.*]] = getelementptr inbounds <16 x i8>, ptr addrspace(3) [[PTR]], i32 0, i64 3
+; GFX12-NEXT: [[E3:%.*]] = load i8, ptr addrspace(3) [[TMP2]], align 1
+; GFX12-NEXT: [[TMP3:%.*]] = getelementptr inbounds <16 x i8>, ptr addrspace(3) [[PTR]], i32 0, i64 5
+; GFX12-NEXT: [[E5:%.*]] = load i8, ptr addrspace(3) [[TMP3]], align 1
+; GFX12-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT:%.*]], i64 3
+; GFX12-NEXT: store i8 [[E1]], ptr addrspace(1) [[P4]], align 1
+; GFX12-NEXT: [[P8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 8
+; GFX12-NEXT: store i8 [[E3]], ptr addrspace(1) [[P8]], align 1
+; GFX12-NEXT: [[P11:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 11
+; GFX12-NEXT: store i8 [[E5]], ptr addrspace(1) [[P11]], align 1
+; GFX12-NEXT: ret void
+;
+ %ptr = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 0
+ %val = load <16 x i8>, ptr addrspace(3) %ptr, align 16
+ %e1 = extractelement <16 x i8> %val, i64 1
+ %e3 = extractelement <16 x i8> %val, i64 3
+ %e5 = extractelement <16 x i8> %val, i64 5
+ %p4 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 3
+ store i8 %e1, ptr addrspace(1) %p4, align 1
+ %p8 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 8
+ store i8 %e3, ptr addrspace(1) %p8, align 1
+ %p11 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 11
+ store i8 %e5, ptr addrspace(1) %p11, align 1
+ ret void
+}
+
+; VectorCombine does not attempt to scalarize loads with extracts of dynamic
+; indices.
+define void @test_v16i8_load_with_dynamic_index_extracts(ptr addrspace(1) %out, i64 %arg) {
+; GFX9-LABEL: @test_v16i8_load_with_dynamic_index_extracts(
+; GFX9-NEXT: [[PTR:%.*]] = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 0
+; GFX9-NEXT: [[VAL:%.*]] = load <16 x i8>, ptr addrspace(3) [[PTR]], align 16
+; GFX9-NEXT: [[E0:%.*]] = extractelement <16 x i8> [[VAL]], i64 [[ARG:%.*]]
+; GFX9-NEXT: [[E1:%.*]] = extractelement <16 x i8> [[VAL]], i64 1
+; GFX9-NEXT: [[E3:%.*]] = extractelement <16 x i8> [[VAL]], i64 3
+; GFX9-NEXT: [[E8:%.*]] = extractelement <16 x i8> [[VAL]], i64 8
+; GFX9-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT:%.*]], i64 0
+; GFX9-NEXT: store i8 [[E0]], ptr addrspace(1) [[P0]], align 1
+; GFX9-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 4
+; GFX9-NEXT: store i8 [[E1]], ptr addrspace(1) [[P4]], align 1
+; GFX9-NEXT: [[P8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 8
+; GFX9-NEXT: store i8 [[E3]], ptr addrspace(1) [[P8]], align 1
+; GFX9-NEXT: [[P11:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 11
+; GFX9-NEXT: store i8 [[E8]], ptr addrspace(1) [[P11]], align 1
+; GFX9-NEXT: ret void
+;
+; GFX12-LABEL: @test_v16i8_load_with_dynamic_index_extracts(
+; GFX12-NEXT: [[PTR:%.*]] = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 0
+; GFX12-NEXT: [[VAL:%.*]] = load <16 x i8>, ptr addrspace(3) [[PTR]], align 16
+; GFX12-NEXT: [[E0:%.*]] = extractelement <16 x i8> [[VAL]], i64 [[ARG:%.*]]
+; GFX12-NEXT: [[E1:%.*]] = extractelement <16 x i8> [[VAL]], i64 1
+; GFX12-NEXT: [[E3:%.*]] = extractelement <16 x i8> [[VAL]], i64 3
+; GFX12-NEXT: [[E8:%.*]] = extractelement <16 x i8> [[VAL]], i64 8
+; GFX12-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT:%.*]], i64 0
+; GFX12-NEXT: store i8 [[E0]], ptr addrspace(1) [[P0]], align 1
+; GFX12-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 4
+; GFX12-NEXT: store i8 [[E1]], ptr addrspace(1) [[P4]], align 1
+; GFX12-NEXT: [[P8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 8
+; GFX12-NEXT: store i8 [[E3]], ptr addrspace(1) [[P8]], align 1
+; GFX12-NEXT: [[P11:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 11
+; GFX12-NEXT: store i8 [[E8]], ptr addrspace(1) [[P11]], align 1
+; GFX12-NEXT: ret void
+;
+ %ptr = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 0
+ %val = load <16 x i8>, ptr addrspace(3) %ptr, align 16
+ %e0 = extractelement <16 x i8> %val, i64 %arg
+ %e1 = extractelement <16 x i8> %val, i64 1
+ %e3 = extractelement <16 x i8> %val, i64 3
+ %e8 = extractelement <16 x i8> %val, i64 8
+ %p0 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 0
+ store i8 %e0, ptr addrspace(1) %p0, align 1
+ %p4 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 4
+ store i8 %e1, ptr addrspace(1) %p4, align 1
+ %p8 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 8
+ store i8 %e3, ptr addrspace(1) %p8, align 1
+ %p11 = getelementptr inbounds i8, ptr addrspace(1) %out, i64 11
+ store i8 %e8, ptr addrspace(1) %p11, align 1
+ ret void
+}
+
+define void @test_v8i8_load_with_extracts_no_scalarize(ptr addrspace(1) %out) {
+; GFX9-LABEL: @test_v8i8_load_with_extracts_no_scalarize(
+; GFX9-NEXT: [[PTR:%.*]] = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 0
+; GFX9-NEXT: [[VAL:%.*]] = load <8 x i8>, ptr addrspace(3) [[PTR]], align 16
+; GFX9-NEXT: [[E0:%.*]] = extractelement <8 x i8> [[VAL]], i64 1
+; GFX9-NEXT: [[E3:%.*]] = extractelement <8 x i8> [[VAL]], i64 3
+; GFX9-NEXT: [[E4:%.*]] = extractelement <8 x i8> [[VAL]], i64 4
+; GFX9-NEXT: [[E6:%.*]] = extractelement <8 x i8> [[VAL]], i64 6
+; GFX9-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT:%.*]], i64 0
+; GFX9-NEXT: store i8 [[E0]], ptr addrspace(1) [[P0]], align 1
+; GFX9-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 2
+; GFX9-NEXT: store i8 [[E3]], ptr addrspace(1) [[P4]], align 1
+; GFX9-NEXT: [[P8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 6
+; GFX9-NEXT: store i8 [[E4]], ptr addrspace(1) [[P8]], align 1
+; GFX9-NEXT: [[P12:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 7
+; GFX9-NEXT: store i8 [[E6]], ptr addrspace(1) [[P12]], align 1
+; GFX9-NEXT: ret void
+;
+; GFX12-LABEL: @test_v8i8_load_with_extracts_no_scalarize(
+; GFX12-NEXT: [[PTR:%.*]] = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 0
+; GFX12-NEXT: [[VAL:%.*]] = load <8 x i8>, ptr addrspace(3) [[PTR]], align 16
+; GFX12-NEXT: [[E0:%.*]] = extractelement <8 x i8> [[VAL]], i64 1
+; GFX12-NEXT: [[E3:%.*]] = extractelement <8 x i8> [[VAL]], i64 3
+; GFX12-NEXT: [[E4:%.*]] = extractelement <8 x i8> [[VAL]], i64 4
+; GFX12-NEXT: [[E6:%.*]] = extractelement <8 x i8> [[VAL]], i64 6
+; GFX12-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT:%.*]], i64 0
+; GFX12-NEXT: store i8 [[E0]], ptr addrspace(1) [[P0]], align 1
+; GFX12-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 2
+; GFX12-NEXT: store i8 [[E3]], ptr addrspace(1) [[P4]], align 1
+; GFX12-NEXT: [[P8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 6
+; GFX12-NEXT: store i8 [[E4]], ptr addrspace(1) [[P8]], align 1
+; GFX12-NEXT: [[P12:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i64 7
+; GFX12-NEXT: store i8 [[E6]], ptr addrspace(1) [[P12]], align 1
+; GFX12-NEXT: ret void
+;
+ %ptr = getelementptr inbounds i8, ptr addrspace(3) @lds, i32 0
+ %val = load <8 x i8>, ptr addrspace(3) %ptr, align 16
+ %e0 = extractelement <8 x i8> %val, i64 1
+ %e3 = extractelement <8 x i8> %val, i64 3
+ %e4 = extractelement <8 x i8> %val, i64 4
+ %e6 = extractelement <8x i8> %val, i64 6
----------------
JanekvO wrote:
```suggestion
%e6 = extractelement <8 x i8> %val, i64 6
```
https://github.com/llvm/llvm-project/pull/194991
More information about the llvm-commits
mailing list