[llvm] [AMDGPU][InstCombine] Optimize constant shuffle patterns (PR #192246)

Matt Arsenault via llvm-commits llvm-commits at lists.llvm.org
Thu Apr 30 04:17:39 PDT 2026


================
@@ -0,0 +1,508 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+wavefrontsize64 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11-W64
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX9
+
+declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)
+declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
+declare i32 @llvm.amdgcn.wave.shuffle.i32(i32, i32)
+declare float @llvm.amdgcn.wave.shuffle.f32(float, i32)
+declare ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3), i32)
+declare i32 @llvm.amdgcn.ds.bpermute(i32, i32)
+
+; Quad perm: lane ^ 1 swaps adjacent pairs.
+define i32 @test_quad_perm_xor1_w32(i32 %val) {
+; GFX11-LABEL: @test_quad_perm_xor1_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor1_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_quad_perm_xor1_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %idx = xor i32 %lane, 1
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; Quad perm: lane ^ 3 reverses within each quad.
+define i32 @test_quad_perm_xor3_w32(i32 %val) {
+; GFX11-LABEL: @test_quad_perm_xor3_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 27, i32 15, i32 15, i1 true)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor3_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 3
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_quad_perm_xor3_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 3
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %idx = xor i32 %lane, 3
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; Half-row mirror: reverse within every 8-lane group.
+define i32 @test_half_row_mirror_w32(i32 %val) {
+; GFX11-LABEL: @test_half_row_mirror_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 321, i32 15, i32 15, i1 true)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_half_row_mirror_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 7
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_half_row_mirror_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 7
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %low3 = and i32 %lane, 7
+  %rev = sub i32 7, %low3
+  %high = and i32 %lane, -8
+  %idx = or i32 %high, %rev
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; Full-row mirror: reverse within every 16-lane row.
+define i32 @test_full_row_mirror_w32(i32 %val) {
+; GFX11-LABEL: @test_full_row_mirror_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 320, i32 15, i32 15, i1 true)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_full_row_mirror_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 15
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_full_row_mirror_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 15
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %low4 = and i32 %lane, 15
+  %rev = sub i32 15, %low4
+  %high = and i32 %lane, -16
+  %idx = or i32 %high, %rev
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; Row rotate right by 3 within each 16-lane row.
+define i32 @test_row_rotate_right3_w32(i32 %val) {
+; GFX11-LABEL: @test_row_rotate_right3_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 301, i32 15, i32 15, i1 true)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_row_rotate_right3_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 3
+; GFX11-W64-NEXT:    [[WRAPPED:%.*]] = and i32 [[ROT]], 15
+; GFX11-W64-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX11-W64-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_row_rotate_right3_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 3
+; GFX9-NEXT:    [[WRAPPED:%.*]] = and i32 [[ROT]], 15
+; GFX9-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX9-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %low4 = and i32 %lane, 15
+  %rot = add i32 %low4, 3
+  %wrapped = and i32 %rot, 15
+  %high = and i32 %lane, -16
+  %idx = or i32 %high, %wrapped
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; Row share: broadcast lane 3 within each 16-lane row.
+define i32 @test_row_share3_w32(i32 %val) {
+; GFX11-LABEL: @test_row_share3_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 339, i32 15, i32 15, i1 true)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_row_share3_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX11-W64-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], 3
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_row_share3_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX9-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], 3
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %high = and i32 %lane, -16
+  %idx = or i32 %high, 3
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; Row xmask: xor each lane with mask 5 within each 16-lane row.
+define i32 @test_row_xmask5_w32(i32 %val) {
+; GFX11-LABEL: @test_row_xmask5_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_row_xmask5_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 5
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_row_xmask5_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 5
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %low4 = and i32 %lane, 15
+  %xored = xor i32 %low4, 5
+  %high = and i32 %lane, -16
+  %idx = or i32 %high, %xored
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; Half-wave swap (wave64 only): lane ^ 32.
+define i32 @test_half_wave_swap_w64(i32 %val) {
+; GFX11-LABEL: @test_half_wave_swap_w64(
+; GFX11-NEXT:    [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-NEXT:    [[IDX:%.*]] = xor i32 [[LO]], 32
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_half_wave_swap_w64(
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlane64.i32(i32 [[VAL:%.*]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_half_wave_swap_w64(
+; GFX9-NEXT:    [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 32
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+  %idx = xor i32 %lane, 32
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; DPP8: rotate left by 1 within each 8-lane group.
+define i32 @test_dpp8_rotate1_w32(i32 %val) {
+; GFX11-LABEL: @test_dpp8_rotate1_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 [[VAL:%.*]], i32 2054353)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_dpp8_rotate1_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 1
+; GFX11-W64-NEXT:    [[WRAPPED:%.*]] = and i32 [[ROT]], 7
+; GFX11-W64-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 56
+; GFX11-W64-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_dpp8_rotate1_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 1
+; GFX9-NEXT:    [[WRAPPED:%.*]] = and i32 [[ROT]], 7
+; GFX9-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 56
+; GFX9-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %low3 = and i32 %lane, 7
+  %rot = add i32 %low3, 1
+  %wrapped = and i32 %rot, 7
+  %high = and i32 %lane, -8
+  %idx = or i32 %high, %wrapped
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; Permlane16: (lane * 3) & 15, a bijection on Z/16Z.
+define i32 @test_permlane16_mul3_w32(i32 %val) {
+; GFX11-LABEL: @test_permlane16_mul3_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlane16.i32(i32 poison, i32 [[VAL:%.*]], i32 1392285232, i32 -629924168, i1 false, i1 false)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_permlane16_mul3_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[MUL3:%.*]] = mul nuw nsw i32 [[LANE]], 3
+; GFX11-W64-NEXT:    [[PERM:%.*]] = and i32 [[MUL3]], 15
+; GFX11-W64-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX11-W64-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[PERM]]
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_permlane16_mul3_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[MUL3:%.*]] = mul nuw nsw i32 [[LANE]], 3
+; GFX9-NEXT:    [[PERM:%.*]] = and i32 [[MUL3]], 15
+; GFX9-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX9-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[PERM]]
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %low4 = and i32 %lane, 15
+  %mul3 = mul i32 %low4, 3
+  %perm = and i32 %mul3, 15
+  %high = and i32 %lane, -16
+  %idx = or i32 %high, %perm
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; PermlaneX16: lane ^ 16. Cross-row swap; identity selectors.
+define i32 @test_permlanex16_xor16_w32(i32 %val) {
+; GFX11-LABEL: @test_permlanex16_xor16_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1985229328, i32 -19088744, i1 false, i1 false)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_permlanex16_xor16_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 16
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_permlanex16_xor16_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 16
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %idx = xor i32 %lane, 16
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; PermlaneX16: lane ^ 17. Cross-row + XOR-by-1 within row.
+define i32 @test_permlanex16_xor17_w32(i32 %val) {
+; GFX11-LABEL: @test_permlanex16_xor17_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1732584193, i32 -271733879, i1 false, i1 false)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_permlanex16_xor17_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 17
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_permlanex16_xor17_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 17
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %idx = xor i32 %lane, 17
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; PermlaneX16: lane ^ 31. Cross-row + reverse within row.
+define i32 @test_permlanex16_xor31_w32(i32 %val) {
+; GFX11-LABEL: @test_permlanex16_xor31_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 -1985229329, i32 19088743, i1 false, i1 false)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_permlanex16_xor31_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 31
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_permlanex16_xor31_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 31
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %idx = xor i32 %lane, 31
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; Float type should work the same as i32.
+define float @test_quad_perm_xor1_float_w32(float %val) {
+; GFX11-LABEL: @test_quad_perm_xor1_float_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float poison, float [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
+; GFX11-NEXT:    ret float [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor1_float_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret float [[RESULT]]
+;
+; GFX9-LABEL: @test_quad_perm_xor1_float_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX9-NEXT:    [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret float [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %idx = xor i32 %lane, 1
+  %result = call float @llvm.amdgcn.wave.shuffle.f32(float %val, i32 %idx)
+  ret float %result
+}
+
+; LDS (32-bit) pointer should optimize the same as i32: the gate must not
+; reject 32-bit pointer types via Type::getPrimitiveSizeInBits().
+define ptr addrspace(3) @test_quad_perm_xor1_ptr3_w32(ptr addrspace(3) %val) {
+; GFX11-LABEL: @test_quad_perm_xor1_ptr3_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.update.dpp.p3(ptr addrspace(3) poison, ptr addrspace(3) [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
+; GFX11-NEXT:    ret ptr addrspace(3) [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor1_ptr3_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret ptr addrspace(3) [[RESULT]]
+;
+; GFX9-LABEL: @test_quad_perm_xor1_ptr3_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX9-NEXT:    [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret ptr addrspace(3) [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %idx = xor i32 %lane, 1
+  %result = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) %val, i32 %idx)
+  ret ptr addrspace(3) %result
+}
+
+; Negative: non-constant index, should not be optimized.
+define i32 @test_nonconstant_shuffle_w32(i32 %val, i32 %idx) {
+; GFX11-LABEL: @test_nonconstant_shuffle_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_nonconstant_shuffle_w32(
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_nonconstant_shuffle_w32(
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; Identity shuffle: lane_id itself.
+define i32 @test_identity_shuffle_w32(i32 %val) {
+; GFX11-LABEL: @test_identity_shuffle_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 228, i32 15, i32 15, i1 true)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_identity_shuffle_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[LANE]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_identity_shuffle_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[LANE]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %lane)
+  ret i32 %result
+}
+
+; Negative: index out of range (lane + 32 >= 32 for wave32).
+define i32 @test_out_of_range_w32(i32 %val) {
+; GFX11-LABEL: @test_out_of_range_w32(
+; GFX11-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-NEXT:    [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_out_of_range_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_out_of_range_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %idx = add i32 %lane, 32
+  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+  ret i32 %result
+}
+
+; ds_bpermute with a constant shuffle pattern (quad perm via byte address).
+define i32 @test_bpermute_quad_xor1_w32(i32 %val) {
+; GFX11-LABEL: @test_bpermute_quad_xor1_w32(
+; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
+; GFX11-NEXT:    ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_bpermute_quad_xor1_w32(
+; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT:    [[IDX:%.*]] = shl nuw nsw i32 [[LANE]], 2
+; GFX11-W64-NEXT:    [[ADDR:%.*]] = xor i32 [[IDX]], 4
+; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[ADDR]], i32 [[VAL:%.*]])
+; GFX11-W64-NEXT:    ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_bpermute_quad_xor1_w32(
+; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT:    [[IDX:%.*]] = shl nuw nsw i32 [[LANE]], 2
+; GFX9-NEXT:    [[ADDR:%.*]] = xor i32 [[IDX]], 4
+; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[ADDR]], i32 [[VAL:%.*]])
+; GFX9-NEXT:    ret i32 [[RESULT]]
+;
+  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %idx = xor i32 %lane, 1
+  %addr = shl i32 %idx, 2
+  %result = call i32 @llvm.amdgcn.ds.bpermute(i32 %addr, i32 %val)
+  ret i32 %result
+}
----------------
arsenm wrote:

Also 64-bit and 16-bit 

https://github.com/llvm/llvm-project/pull/192246


More information about the llvm-commits mailing list