[llvm] [SPIRV] Respect source alignment when lowering memory copies (PR #228859)
Alexander Shaposhnikov via llvm-commits
llvm-commits at lists.llvm.org
Sun Oct 4 11:34:35 PDT 2026
https://github.com/alexander-shaposhnikov updated https://github.com/llvm/llvm-project/pull/228859
>From 013c293632cfe3278646c107687b01da02af72d5 Mon Sep 17 00:00:00 2001
From: Alexander Shaposhnikov <alexander.v.shaposhnikov at gmail.com>
Date: Sun, 4 Oct 2026 11:33:28 -0700
Subject: [PATCH] [SPIRV] Respect source alignment when lowering memory copies
Lowering memcpy, memcpy.inline, and memmove to OpCopyMemory or
OpCopyMemorySized only considers the destination alignment. A single
Aligned memory operand applies to both pointers, so a destination
alignment greater than the source alignment overstates the latter.
For SPIR-V 1.4 and later, emit separate destination and source alignments
when they differ. For earlier versions, use their minimum in the single
permitted memory operand mask. Keep one mask when the alignments match.
Consolidate both copy selectors in selectCopyMemory so they share the
alignment handling. Allow the memory operand helper to take an explicit
alignment while preserving the existing flags and alias metadata on both
accesses.
Cover equal and unequal alignments in SPIR-V 1.3 and 1.4, and add an
OpCopyMemory regression case with unequal alignments, volatility, and
alias-scope metadata.
---
.../Target/SPIRV/SPIRVInstructionSelector.cpp | 93 ++++++++++---------
.../alias-copy-memory.ll | 6 +-
.../SPIRV/llvm-intrinsics/memcpy.align.ll | 16 ++--
3 files changed, 63 insertions(+), 52 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 143e1c0e3ef2a..9cf72523b45a1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -168,7 +168,6 @@ class SPIRVInstructionSelector : public InstructionSelector {
bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
Register getOrCreateMemSetGlobal(MachineInstr &I) const;
bool selectCopyMemory(MachineInstr &I, Register SrcReg) const;
- bool selectCopyMemorySized(MachineInstr &I, Register SrcReg) const;
bool selectAtomicRMW(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I,
unsigned NewOpcode, unsigned NegateOpcode = 0) const;
@@ -1969,10 +1968,10 @@ bool SPIRVInstructionSelector::selectBitcast(Register ResVReg,
return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
}
-static void addMemoryOperands(MachineMemOperand *MemOp,
- MachineInstrBuilder &MIB,
- MachineIRBuilder &MIRBuilder,
- SPIRVGlobalRegistry &GR) {
+static void
+addMemoryOperands(const MachineMemOperand *MemOp, MachineInstrBuilder &MIB,
+ MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry &GR,
+ std::optional<Align> Align = std::nullopt) {
const SPIRVSubtarget *ST =
static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
@@ -2004,7 +2003,7 @@ static void addMemoryOperands(MachineMemOperand *MemOp,
if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
MIB.addImm(SpvMemOp);
if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
- MIB.addImm(MemOp->getAlign().value());
+ MIB.addImm(Align.value_or(MemOp->getAlign()).value());
if (AliasList)
MIB.addUse(AliasList->getOperand(0).getReg());
if (NoAliasList)
@@ -2535,42 +2534,53 @@ bool SPIRVInstructionSelector::selectCopyMemory(MachineInstr &I,
Register SrcReg) const {
MachineBasicBlock &BB = *I.getParent();
Register DstReg = I.getOperand(0).getReg();
- SPIRVTypeInst DstTy = GR.getSPIRVTypeForVReg(DstReg);
- SPIRVTypeInst SrcTy = GR.getSPIRVTypeForVReg(SrcReg);
- if (GR.getPointeeType(DstTy) != GR.getPointeeType(SrcTy))
- return diagnoseUnsupported(
- I, "OpCopyMemory requires operands to have the same type");
- uint64_t CopySize = getIConstVal(I.getOperand(2).getReg(), MRI);
- SPIRVTypeInst PointeeTy = GR.getPointeeType(DstTy);
- const Type *LLVMPointeeTy = GR.getTypeForSPIRVType(PointeeTy);
- if (!LLVMPointeeTy)
- return diagnoseUnsupported(
- I, "Unable to determine pointee type size for OpCopyMemory");
- const DataLayout &DL = I.getMF()->getFunction().getDataLayout();
- if (CopySize != DL.getTypeStoreSize(const_cast<Type *>(LLVMPointeeTy)))
- return diagnoseUnsupported(
- I, "OpCopyMemory requires the size to match the pointee type size");
- auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemory))
+ const bool IsLogical = STI.isLogicalSPIRV();
+ if (IsLogical) {
+ SPIRVTypeInst DstTy = GR.getSPIRVTypeForVReg(DstReg);
+ SPIRVTypeInst SrcTy = GR.getSPIRVTypeForVReg(SrcReg);
+ if (GR.getPointeeType(DstTy) != GR.getPointeeType(SrcTy))
+ return diagnoseUnsupported(
+ I, "OpCopyMemory requires operands to have the same type");
+ uint64_t CopySize = getIConstVal(I.getOperand(2).getReg(), MRI);
+ SPIRVTypeInst PointeeTy = GR.getPointeeType(DstTy);
+ const Type *LLVMPointeeTy = GR.getTypeForSPIRVType(PointeeTy);
+ if (!LLVMPointeeTy)
+ return diagnoseUnsupported(
+ I, "Unable to determine pointee type size for OpCopyMemory");
+ const DataLayout &DL = I.getMF()->getFunction().getDataLayout();
+ if (CopySize != DL.getTypeStoreSize(const_cast<Type *>(LLVMPointeeTy)))
+ return diagnoseUnsupported(
+ I, "OpCopyMemory requires the size to match the pointee type size");
+ }
+
+ const unsigned Opcode =
+ IsLogical ? SPIRV::OpCopyMemory : SPIRV::OpCopyMemorySized;
+ auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
.addUse(DstReg)
.addUse(SrcReg);
- if (I.getNumMemOperands()) {
- MachineIRBuilder MIRBuilder(I);
- addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
- }
- MIB.constrainAllUses(TII, TRI, RBI);
- return true;
-}
+ if (!IsLogical)
+ MIB.addUse(I.getOperand(2).getReg());
-bool SPIRVInstructionSelector::selectCopyMemorySized(MachineInstr &I,
- Register SrcReg) const {
- MachineBasicBlock &BB = *I.getParent();
- auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized))
- .addUse(I.getOperand(0).getReg())
- .addUse(SrcReg)
- .addUse(I.getOperand(2).getReg());
if (I.getNumMemOperands()) {
MachineIRBuilder MIRBuilder(I);
- addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR);
+ const MachineMemOperand *DstMemOp = *I.memoperands_begin();
+ const Align DstAlign = DstMemOp->getAlign();
+ Align SrcAlign = DstAlign;
+ // Copies carry destination and source MMOs; memset carries only the
+ // destination. Shader memory operands do not include alignment.
+ if (I.getNumMemOperands() > 1 && !STI.isShader())
+ SrcAlign = (*std::next(I.memoperands_begin()))->getAlign();
+
+ // A single mask applies to both pointers. SPIR-V 1.4 allows separate
+ // destination and source masks, preserving their individual alignments.
+ if (DstAlign != SrcAlign && STI.isAtLeastSPIRVVer(VersionTuple(1, 4))) {
+ // Preserve the existing flags and metadata on both accesses.
+ addMemoryOperands(DstMemOp, MIB, MIRBuilder, GR);
+ addMemoryOperands(DstMemOp, MIB, MIRBuilder, GR, SrcAlign);
+ } else {
+ addMemoryOperands(DstMemOp, MIB, MIRBuilder, GR,
+ std::min(DstAlign, SrcAlign));
+ }
}
MIB.constrainAllUses(TII, TRI, RBI);
return true;
@@ -2598,13 +2608,8 @@ bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
if (!selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast))
return false;
}
- if (STI.isLogicalSPIRV()) {
- if (!selectCopyMemory(I, SrcReg))
- return false;
- } else {
- if (!selectCopyMemorySized(I, SrcReg))
- return false;
- }
+ if (!selectCopyMemory(I, SrcReg))
+ return false;
if (ResVReg.isValid() && ResVReg != I.getOperand(0).getReg())
if (!BuildCOPY(ResVReg, I.getOperand(0).getReg(), I))
return false;
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_memory_access_aliasing/alias-copy-memory.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_memory_access_aliasing/alias-copy-memory.ll
index ddd61bd9e0bbb..6ceddae59ba0d 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_memory_access_aliasing/alias-copy-memory.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_memory_access_aliasing/alias-copy-memory.ll
@@ -9,14 +9,16 @@
; CHECK: %[[#]] = OpFunction
define void @copy_aliased() {
entry:
-; CHECK: OpCopyMemory %[[#]] %[[#]] Aligned|AliasScopeINTELMask 4 %[[#List]]
+; CHECK: OpCopyMemory %[[#]] %[[#]] Aligned|AliasScopeINTELMask 4 %[[#List]]{{$}}
call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) align 4 @dst, ptr addrspace(1) align 4 @src, i64 20, i1 false), !alias.scope !1
+; CHECK: OpCopyMemory %[[#]] %[[#]] Volatile|Aligned|AliasScopeINTELMask 8 %[[#List]] Volatile|Aligned|AliasScopeINTELMask 4 %[[#List]]{{$}}
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) align 8 @dst, ptr addrspace(1) align 4 @src, i64 20, i1 true), !alias.scope !1
ret void
}
%struct.T = type <{ float, float, float, float, float }>
@src = external dso_local addrspace(1) global %struct.T, align 4
- at dst = external dso_local addrspace(1) global %struct.T, align 4
+ at dst = external dso_local addrspace(1) global %struct.T, align 8
declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1), ptr addrspace(1), i64, i1)
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/memcpy.align.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/memcpy.align.ll
index b4100019a8a0c..03f20f0176d2b 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/memcpy.align.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/memcpy.align.ll
@@ -1,5 +1,7 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64v1.3-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,SPV13
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64v1.3-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64v1.4-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,SPV14
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64v1.4-unknown-unknown %s -o - -filetype=obj | spirv-val %}
%struct.B = type { [2 x i32] }
%struct.A = type { i64, %struct.B }
@@ -14,13 +16,14 @@ entry:
call void @llvm.lifetime.start.p0(i64 8, ptr %0)
%1 = bitcast ptr %b to ptr
call void @llvm.memcpy.p0.p2.i32(ptr align 4 %1, ptr addrspace(2) align 4 @__const.foo.b, i32 8, i1 false)
-; CHECK: OpCopyMemorySized %[[#]] %[[#]] %[[#]] Aligned 4
+; CHECK: OpCopyMemorySized %[[#]] %[[#]] %[[#]] Aligned 4{{$}}
%b1 = getelementptr inbounds %struct.A, ptr %agg.result, i32 0, i32 1
%2 = bitcast ptr %b1 to ptr
%3 = bitcast ptr %b to ptr
call void @llvm.memcpy.p0.p0.i32(ptr align 8 %2, ptr align 4 %3, i32 8, i1 false)
; CHECK: %[[#PTR1:]] = OpInBoundsPtrAccessChain %[[#]] %[[#]] %[[#]] %[[#]]
-; CHECK: OpCopyMemorySized %[[#PTR1]] %[[#]] %[[#]] Aligned 8
+; SPV13: OpCopyMemorySized %[[#PTR1]] %[[#]] %[[#]] Aligned 4{{$}}
+; SPV14: OpCopyMemorySized %[[#PTR1]] %[[#]] %[[#]] Aligned 8 Aligned 4{{$}}
%4 = bitcast ptr %b to ptr
call void @llvm.lifetime.end.p0(i64 8, ptr %4)
ret void
@@ -41,13 +44,14 @@ entry:
call void @llvm.lifetime.start.p0(i64 16, ptr %0)
%1 = bitcast ptr %a to ptr
call void @llvm.memcpy.p0.p2.i32(ptr align 8 %1, ptr addrspace(2) align 8 @__const.bar.a, i32 16, i1 false)
-; CHECK: OpCopyMemorySized %[[#]] %[[#]] %[[#]] Aligned 8
+; CHECK: OpCopyMemorySized %[[#]] %[[#]] %[[#]] Aligned 8{{$}}
%b = getelementptr inbounds %struct.A, ptr %a, i32 0, i32 1
%2 = bitcast ptr %agg.result to ptr
%3 = bitcast ptr %b to ptr
call void @llvm.memcpy.p0.p0.i32(ptr align 4 %2, ptr align 8 %3, i32 8, i1 false)
; CHECK: %[[#PTR2:]] = OpInBoundsPtrAccessChain %[[#]] %[[#]] %[[#]] %[[#]]
-; CHECK: OpCopyMemorySized %[[#]] %[[#PTR2]] %[[#]] Aligned 4
+; SPV13: OpCopyMemorySized %[[#]] %[[#PTR2]] %[[#]] Aligned 4{{$}}
+; SPV14: OpCopyMemorySized %[[#]] %[[#PTR2]] %[[#]] Aligned 4 Aligned 8{{$}}
%4 = bitcast ptr %a to ptr
call void @llvm.lifetime.end.p0(i64 16, ptr %4)
ret void
More information about the llvm-commits
mailing list