[clang] [CIR] Add invariant attribute to cir.load (PR #204283)
via cfe-commits
cfe-commits at lists.llvm.org
Tue Jun 23 00:14:50 PDT 2026
https://github.com/skc7 updated https://github.com/llvm/llvm-project/pull/204283
>From 6d3caa24d62bb992d2d639984027c79b48393041 Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Wed, 17 Jun 2026 10:18:51 +0530
Subject: [PATCH 1/2] [CIR] Add invariant attribute to cir.load
---
.../CIR/Dialect/Builder/CIRBaseBuilder.h | 10 +++++----
clang/include/clang/CIR/Dialect/IR/CIROps.td | 10 ++++++++-
clang/lib/CIR/CodeGen/CIRGenBuilder.h | 6 ++++--
.../lib/CIR/Dialect/Transforms/FlattenCFG.cpp | 6 ++++--
.../TargetLowering/CIRABIRewriteContext.cpp | 3 ++-
.../TargetLowering/LowerItaniumCXXABI.cpp | 15 ++++++++-----
.../CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp | 2 +-
clang/test/CIR/IR/load-invariant.cir | 20 ++++++++++++++++++
clang/test/CIR/Lowering/load-invariant.cir | 21 +++++++++++++++++++
9 files changed, 77 insertions(+), 16 deletions(-)
create mode 100644 clang/test/CIR/IR/load-invariant.cir
create mode 100644 clang/test/CIR/Lowering/load-invariant.cir
diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index 0db205f8d5b79..7cedc8bcfab6f 100644
--- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -230,11 +230,12 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
cir::LoadOp createLoad(mlir::Location loc, mlir::Value ptr,
bool isVolatile = false, uint64_t alignment = 0,
- bool isNontemporal = false) {
+ bool isNontemporal = false,
+ bool isInvariant = false) {
mlir::IntegerAttr alignmentAttr = getAlignmentAttr(alignment);
return cir::LoadOp::create(*this, loc, ptr, /*isDeref=*/false, isVolatile,
- isNontemporal, alignmentAttr,
- cir::SyncScopeKindAttr{}, cir::MemOrderAttr{});
+ isNontemporal, alignmentAttr, cir::SyncScopeKindAttr{},
+ cir::MemOrderAttr{}, isInvariant);
}
mlir::Value createAlignedLoad(mlir::Location loc, mlir::Value ptr,
@@ -429,7 +430,8 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
return cir::LoadOp::create(*this, loc, addr, /*isDeref=*/false,
/*isVolatile=*/false, /*nontemporal=*/false,
alignmentAttr,
- /*sync_scope=*/{}, /*mem_order=*/{});
+ /*sync_scope=*/{}, /*mem_order=*/{},
+ /*invariant=*/false);
}
cir::PtrStrideOp createPtrStride(mlir::Location loc, mlir::Value base,
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index f4f22cd297ea6..5cc7c964abfa1 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -671,6 +671,9 @@ def CIR_LoadOp : CIR_Op<"load", [
`alignment` can be used to specify an alignment that's different from the
default, which is computed from `result`'s type ABI data layout.
+ A unit attribute `invariant` can be used to indicate that the loaded memory
+ never changes, mapping to LLVM IR's `!invariant.load` metadata.
+
Example:
```
@@ -685,6 +688,9 @@ def CIR_LoadOp : CIR_Op<"load", [
// Perform a volatile load from address in %0.
%4 = cir.load volatile %0 : !cir.ptr<i32>, i32
+ // Perform an invariant load from address in %0.
+ %5 = cir.load invariant %0 : !cir.ptr<i32>, i32
+
// Others
%x = cir.load align(16) atomic(seq_cst) %0 : !cir.ptr<i32>, i32
```
@@ -697,13 +703,15 @@ def CIR_LoadOp : CIR_Op<"load", [
UnitAttr:$is_nontemporal,
OptionalAttr<I64Attr>:$alignment,
OptionalAttr<CIR_SyncScopeKind>:$sync_scope,
- OptionalAttr<CIR_MemOrder>:$mem_order);
+ OptionalAttr<CIR_MemOrder>:$mem_order,
+ UnitAttr:$invariant);
let results = (outs CIR_AnyType:$result);
let assemblyFormat = [{
(`deref` $isDeref^)?
(`volatile` $is_volatile^)?
(`nontemporal` $is_nontemporal^)?
+ (`invariant` $invariant^)?
(`align` `(` $alignment^ `)`)?
(`syncscope` `(` $sync_scope^ `)`)?
(`atomic` `(` $mem_order^ `)`)?
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.h b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
index b8db0d9157aa6..cc1ff1e32521f 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuilder.h
+++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
@@ -583,7 +583,8 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
isVolatile, isNontemporal,
/*alignment=*/align,
/*sync_scope=*/cir::SyncScopeKindAttr{},
- /*mem_order=*/cir::MemOrderAttr{});
+ /*mem_order=*/cir::MemOrderAttr{},
+ /*invariant=*/false);
}
cir::LoadOp createAlignedLoad(mlir::Location loc, mlir::Type ty,
@@ -596,7 +597,8 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
/*isVolatile=*/false, /*isNontemporal=*/false,
alignAttr,
/*sync_scope=*/cir::SyncScopeKindAttr{},
- /*mem_order=*/cir::MemOrderAttr{});
+ /*mem_order=*/cir::MemOrderAttr{},
+ /*invariant=*/false);
}
cir::LoadOp
diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
index c487e645e30cd..57507240ddfd8 100644
--- a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
@@ -982,7 +982,8 @@ class CIRCleanupScopeOpFlattening
cir::LoadOp::create(rewriter, loc, alloca, /*isDeref=*/false,
/*isVolatile=*/false, /*isNontemporal=*/false,
/*alignment=*/mlir::IntegerAttr(),
- cir::SyncScopeKindAttr(), cir::MemOrderAttr());
+ cir::SyncScopeKindAttr(), cir::MemOrderAttr(),
+ /*invariant=*/false);
returnValues.push_back(loaded);
}
}
@@ -1296,7 +1297,8 @@ class CIRCleanupScopeOpFlattening
cir::LoadOp::create(rewriter, loc, destSlot, /*isDeref=*/false,
/*isVolatile=*/false, /*isNontemporal=*/false,
/*alignment=*/mlir::IntegerAttr(),
- cir::SyncScopeKindAttr(), cir::MemOrderAttr());
+ cir::SyncScopeKindAttr(), cir::MemOrderAttr()
+ /*invariant=*/false);
// Create destination blocks for each exit and collect switch case info.
llvm::SmallVector<mlir::APInt, 8> caseValues;
diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
index b252ed188c408..00a41456b303b 100644
--- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
@@ -534,7 +534,8 @@ void rewriteIndirectReturnCall(cir::CallOp call,
/*isVolatile=*/mlir::UnitAttr(),
/*alignment=*/mlir::IntegerAttr(),
/*sync_scope=*/cir::SyncScopeKindAttr(),
- /*mem_order=*/cir::MemOrderAttr());
+ /*mem_order=*/cir::MemOrderAttr(),
+ /*invariant=*/mlir::UnitAttr());
call.getResult().replaceAllUsesWith(load);
}
call->erase();
diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/LowerItaniumCXXABI.cpp b/clang/lib/CIR/Dialect/Transforms/TargetLowering/LowerItaniumCXXABI.cpp
index 6e12a13787a2a..5218d4979f353 100644
--- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/LowerItaniumCXXABI.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/LowerItaniumCXXABI.cpp
@@ -385,7 +385,8 @@ void LowerItaniumCXXABI::lowerGetMethod(
/*isNontemporal=*/false,
/*alignment=*/mlir::IntegerAttr(),
/*sync_scope=*/cir::SyncScopeKindAttr{},
- /*mem_order=*/cir::MemOrderAttr());
+ /*mem_order=*/cir::MemOrderAttr(),
+ /*invariant=*/false);
// Apply the offset.
// On ARM64, to reserve extra space in virtual member function pointers,
@@ -413,7 +414,8 @@ void LowerItaniumCXXABI::lowerGetMethod(
/*isNontemporal=*/false,
/*alignment=*/mlir::IntegerAttr(),
/*sync_scope=*/cir::SyncScopeKindAttr{},
- /*mem_order=*/cir::MemOrderAttr());
+ /*mem_order=*/cir::MemOrderAttr(),
+ /*invariant=*/false);
cir::YieldOp::create(b, loc, fnPtr.getResult());
assert(!cir::MissingFeatures::emitCFICheck());
@@ -787,7 +789,8 @@ static mlir::Value buildDynamicCastToVoidAfterNullCheck(
/*isNontemporal=*/false,
/*alignment=*/builder.getI64IntegerAttr(vtableElemAlign),
/*sync_scope=*/cir::SyncScopeKindAttr(),
- /*mem_order=*/cir::MemOrderAttr());
+ /*mem_order=*/cir::MemOrderAttr(),
+ /*invariant=*/false);
mlir::Value elementPtr = cir::CastOp::create(builder, loc, vtableElemPtrTy,
cir::CastKind::bitcast, vptr);
mlir::Value minusTwo =
@@ -801,7 +804,8 @@ static mlir::Value buildDynamicCastToVoidAfterNullCheck(
/*isNontemporal=*/false,
/*alignment=*/builder.getI64IntegerAttr(vtableElemAlign),
/*sync_scope=*/cir::SyncScopeKindAttr(),
- /*mem_order=*/cir::MemOrderAttr());
+ /*mem_order=*/cir::MemOrderAttr(),
+ /*invariant=*/false);
auto voidPtrTy =
cir::PointerType::get(cir::VoidType::get(builder.getContext()));
@@ -910,7 +914,8 @@ mlir::Value LowerItaniumCXXABI::readArrayCookieImpl(
builder, loc, countPtr, /*isDeref=*/false, /*isVolatile=*/false,
/*isNontemporal=*/false,
builder.getI64IntegerAttr(countAlignment.getQuantity()),
- cir::SyncScopeKindAttr(), cir::MemOrderAttr());
+ cir::SyncScopeKindAttr(), cir::MemOrderAttr(),
+ /*invariant=*/false);
}
} // namespace cir
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 27eba4ee326a5..199b3f0d2233d 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -1866,7 +1866,7 @@ mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
mlir::LLVM::LoadOp newLoad = mlir::LLVM::LoadOp::create(
rewriter, op->getLoc(), llvmTy, adaptor.getAddr(), alignment,
op.getIsVolatile(), /*isNonTemporal=*/op.getIsNontemporal(),
- /*isInvariant=*/false, /*isInvariantGroup=*/false, ordering,
+ /*isInvariant=*/op.getInvariant(), /*isInvariantGroup=*/false, ordering,
llvmSyncScope.value_or(std::string()));
// Convert adapted result to its original type if needed.
diff --git a/clang/test/CIR/IR/load-invariant.cir b/clang/test/CIR/IR/load-invariant.cir
new file mode 100644
index 0000000000000..4b014666db3a9
--- /dev/null
+++ b/clang/test/CIR/IR/load-invariant.cir
@@ -0,0 +1,20 @@
+// RUN: cir-opt %s --verify-roundtrip | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+
+module {
+ // CHECK-LABEL: cir.func @invariant_load
+ // CHECK: %{{.*}} = cir.load invariant %arg0 : !cir.ptr<!s32i>, !s32i
+ cir.func @invariant_load(%arg0: !cir.ptr<!s32i>) -> !s32i {
+ %0 = cir.load invariant %arg0 : !cir.ptr<!s32i>, !s32i
+ cir.return %0 : !s32i
+ }
+
+ // A plain load must not print the invariant keyword.
+ // CHECK-LABEL: cir.func @plain_load
+ // CHECK: %{{.*}} = cir.load %arg0 : !cir.ptr<!s32i>, !s32i
+ cir.func @plain_load(%arg0: !cir.ptr<!s32i>) -> !s32i {
+ %0 = cir.load %arg0 : !cir.ptr<!s32i>, !s32i
+ cir.return %0 : !s32i
+ }
+}
diff --git a/clang/test/CIR/Lowering/load-invariant.cir b/clang/test/CIR/Lowering/load-invariant.cir
new file mode 100644
index 0000000000000..40c929ba227df
--- /dev/null
+++ b/clang/test/CIR/Lowering/load-invariant.cir
@@ -0,0 +1,21 @@
+// RUN: cir-opt %s -cir-to-llvm -o - | FileCheck %s
+
+// The invariant flag on cir.load lowers to the LLVM dialect invariant load.
+
+!s32i = !cir.int<s, 32>
+
+module {
+ // CHECK-LABEL: llvm.func @invariant_load
+ // CHECK: llvm.load %{{.*}} invariant {{.*}}: !llvm.ptr -> i32
+ cir.func @invariant_load(%arg0: !cir.ptr<!s32i>) -> !s32i {
+ %0 = cir.load invariant %arg0 : !cir.ptr<!s32i>, !s32i
+ cir.return %0 : !s32i
+ }
+
+ // CHECK-LABEL: llvm.func @plain_load
+ // CHECK-NOT: invariant
+ cir.func @plain_load(%arg0: !cir.ptr<!s32i>) -> !s32i {
+ %0 = cir.load %arg0 : !cir.ptr<!s32i>, !s32i
+ cir.return %0 : !s32i
+ }
+}
>From fc6603e59ffe89aec698efa6751a5860e63e16fa Mon Sep 17 00:00:00 2001
From: skc7 <Krishna.Sankisa at amd.com>
Date: Tue, 23 Jun 2026 12:30:13 +0530
Subject: [PATCH 2/2] set it on virtual call loads
---
.../CIR/Dialect/Builder/CIRBaseBuilder.h | 8 +++----
clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp | 11 +++++----
.../lib/CIR/Dialect/Transforms/FlattenCFG.cpp | 2 +-
.../CIR/CodeGen/vtable-load-invariant.cpp | 23 +++++++++++++++++++
4 files changed, 34 insertions(+), 10 deletions(-)
create mode 100644 clang/test/CIR/CodeGen/vtable-load-invariant.cpp
diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index 7cedc8bcfab6f..423630858f0a6 100644
--- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -230,12 +230,12 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
cir::LoadOp createLoad(mlir::Location loc, mlir::Value ptr,
bool isVolatile = false, uint64_t alignment = 0,
- bool isNontemporal = false,
- bool isInvariant = false) {
+ bool isNontemporal = false) {
mlir::IntegerAttr alignmentAttr = getAlignmentAttr(alignment);
return cir::LoadOp::create(*this, loc, ptr, /*isDeref=*/false, isVolatile,
- isNontemporal, alignmentAttr, cir::SyncScopeKindAttr{},
- cir::MemOrderAttr{}, isInvariant);
+ isNontemporal, alignmentAttr,
+ cir::SyncScopeKindAttr{}, cir::MemOrderAttr{},
+ /*invariant=*/false);
}
mlir::Value createAlignedLoad(mlir::Location loc, mlir::Value ptr,
diff --git a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
index 552d73966e97b..f97538e47d871 100644
--- a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
@@ -1960,16 +1960,17 @@ CIRGenCallee CIRGenItaniumCXXABI::getVirtualFunctionPointer(
cgf.getPointerAlign());
}
- // Add !invariant.load md to virtual function load to indicate that
- // function didn't change inside vtable.
+ // Set invariant on the cir.load of virtual function pointer to indicate
+ // that function didn't change inside vtable.
// It's safe to add it without -fstrict-vtable-pointers, but it would not
// help in devirtualization because it will only matter if we will have 2
// the same virtual function loads from the same vtable load, which won't
// happen without enabled devirtualization with -fstrict-vtable-pointers.
if (cgm.getCodeGenOpts().OptimizationLevel > 0 &&
- cgm.getCodeGenOpts().StrictVTablePointers) {
- cgm.errorNYI(loc, "getVirtualFunctionPointer: strictVTablePointers");
- }
+ cgm.getCodeGenOpts().StrictVTablePointers)
+ if (auto loadOp = vfuncLoad.getDefiningOp<cir::LoadOp>())
+ loadOp.setInvariant(true);
+
vfunc = vfuncLoad;
}
diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
index 57507240ddfd8..0b8e6b57c6e7a 100644
--- a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
@@ -1297,7 +1297,7 @@ class CIRCleanupScopeOpFlattening
cir::LoadOp::create(rewriter, loc, destSlot, /*isDeref=*/false,
/*isVolatile=*/false, /*isNontemporal=*/false,
/*alignment=*/mlir::IntegerAttr(),
- cir::SyncScopeKindAttr(), cir::MemOrderAttr()
+ cir::SyncScopeKindAttr(), cir::MemOrderAttr(),
/*invariant=*/false);
// Create destination blocks for each exit and collect switch case info.
diff --git a/clang/test/CIR/CodeGen/vtable-load-invariant.cpp b/clang/test/CIR/CodeGen/vtable-load-invariant.cpp
new file mode 100644
index 0000000000000..afbe07b6483e5
--- /dev/null
+++ b/clang/test/CIR/CodeGen/vtable-load-invariant.cpp
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O1 -fstrict-vtable-pointers -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --check-prefix=CIR-STRICT --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t-plain.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t-plain.cir %s
+
+struct A {
+ virtual void f(char);
+};
+
+void f1(A *a) {
+ a->f('c');
+}
+
+// Under -O1 -fstrict-vtable-pointers the virtual function pointer load is
+// marked invariant; the vptr load preceding it is not.
+// CIR-STRICT-LABEL: cir.func{{.*}}@_Z2f1P1A
+// CIR-STRICT: %[[VPTR:.*]] = cir.load{{.*}} %{{.*}} : !cir.ptr<!cir.vptr>, !cir.vptr
+// CIR-STRICT: %[[FN_PTR_PTR:.*]] = cir.vtable.get_virtual_fn_addr %[[VPTR]][0]
+// CIR-STRICT: %[[FN_PTR:.*]] = cir.load invariant {{.*}} %[[FN_PTR_PTR]]
+
+// Without the flags the same load must not be invariant.
+// CIR-LABEL: cir.func{{.*}}@_Z2f1P1A
+// CIR-NOT: cir.load invariant
More information about the cfe-commits
mailing list