[clang] [Clang][OpenCL] Mark scalar loads from constant address space invariant (PR #218356)
Keshav Vinayak Jha via cfe-commits
cfe-commits at lists.llvm.org
Sun Nov 8 23:33:28 PST 2026
https://github.com/keshavvinayak01 updated https://github.com/llvm/llvm-project/pull/218356
>From 99438e164b8977db5bbb0eed3077bf71290fcefd Mon Sep 17 00:00:00 2001
From: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
Date: Mon, 24 Aug 2026 09:11:24 +0000
Subject: [PATCH 1/7] [Clang][OpenCL] Mark constant address space loads
invariant
---
clang/lib/CodeGen/CGExpr.cpp | 3 +++
clang/test/CodeGenOpenCL/invariant-load.cl | 24 ++++++++++++++++++++++
2 files changed, 27 insertions(+)
create mode 100644 clang/test/CodeGenOpenCL/invariant-load.cl
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index eff6a7de320d71..ef66b9f0d4af7a 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -2248,6 +2248,9 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
Addr.withElementType(convertTypeForLoadStore(Ty, Addr.getElementType()));
llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile);
+ if (Ty.getAddressSpace() == LangAS::opencl_constant)
+ Load->setMetadata(llvm::LLVMContext::MD_invariant_load,
+ llvm::MDNode::get(Load->getContext(), {}));
if (isNontemporal) {
llvm::MDNode *Node = llvm::MDNode::get(
Load->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
diff --git a/clang/test/CodeGenOpenCL/invariant-load.cl b/clang/test/CodeGenOpenCL/invariant-load.cl
new file mode 100644
index 00000000000000..717348398e28e6
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/invariant-load.cl
@@ -0,0 +1,24 @@
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -cl-std=CL2.0 -O0 -emit-llvm -o - %s | FileCheck %s --check-prefix=AMDGCN
+// RUN: %clang_cc1 -triple spir64-unknown-unknown -cl-std=CL2.0 -O0 -emit-llvm -o - %s | FileCheck %s --check-prefix=SPIR
+
+kernel void constant_load(global int *out, constant int *in) {
+ out[0] = in[0];
+}
+
+// AMDGCN-LABEL: define{{.*}}@constant_load(
+// AMDGCN: load i32, ptr addrspace(4) %{{.*}}, align 4, !invariant.load [[INVARIANT:![0-9]+]]
+// SPIR-LABEL: define{{.*}}@constant_load(
+// SPIR: load i32, ptr addrspace(2) %{{.*}}, align 4, !invariant.load [[INVARIANT:![0-9]+]]
+
+kernel void global_const_load(global int *out, global const int *in) {
+ out[0] = in[0];
+}
+
+// AMDGCN-LABEL: define{{.*}}@global_const_load(
+// AMDGCN: load i32, ptr addrspace(1) %{{.*}}, align 4{{$}}
+// SPIR-LABEL: define{{.*}}@global_const_load(
+// SPIR: load i32, ptr addrspace(1) %{{.*}}, align 4{{$}}
+
+// AMDGCN: [[INVARIANT]] = !{}
+// SPIR: [[INVARIANT]] = !{}
>From c892aff09c611c92fb29b9a3eceb88a62c1ef818 Mon Sep 17 00:00:00 2001
From: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
Date: Mon, 7 Sep 2026 08:17:13 +0000
Subject: [PATCH 2/7] [Clang][CUDA] Mark constant memory loads invariant
Recover the underlying global's target address space for CUDA and HIP device loads, whose source types remain in the default address space. Add CUDA coverage and simplify the OpenCL test as requested in review.
---
clang/lib/CodeGen/CGExpr.cpp | 19 +++++++++-
clang/test/CodeGenCUDA/invariant-load.cu | 43 ++++++++++++++++++++++
clang/test/CodeGenOpenCL/invariant-load.cl | 19 +++-------
3 files changed, 67 insertions(+), 14 deletions(-)
create mode 100644 clang/test/CodeGenCUDA/invariant-load.cu
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index ef66b9f0d4af7a..845db06a4a0ddc 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -42,6 +42,7 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/StringExtras.h"
+#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Intrinsics.h"
@@ -2074,6 +2075,22 @@ llvm::Value *CodeGenFunction::emitScalarConstant(
return Constant.getValue();
}
+static bool isInvariantLoad(CodeGenFunction &CGF, LangAS TypeAS,
+ llvm::Value *Ptr) {
+ if (TypeAS == LangAS::opencl_constant)
+ return true;
+
+ // CUDA represents constant memory as a declaration attribute, so the
+ // expression type uses the default address space. Recover the storage
+ // address space from the underlying global after the address-space cast.
+ if (!CGF.getLangOpts().CUDAIsDevice)
+ return false;
+ const auto *GV =
+ llvm::dyn_cast<llvm::GlobalValue>(llvm::getUnderlyingObject(Ptr));
+ return GV && GV->getAddressSpace() == CGF.getContext().getTargetAddressSpace(
+ LangAS::cuda_constant);
+}
+
llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
SourceLocation Loc) {
return EmitLoadOfScalar(lvalue.getAddress(), lvalue.isVolatile(),
@@ -2248,7 +2265,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
Addr.withElementType(convertTypeForLoadStore(Ty, Addr.getElementType()));
llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile);
- if (Ty.getAddressSpace() == LangAS::opencl_constant)
+ if (isInvariantLoad(*this, Ty.getAddressSpace(), Addr.getBasePointer()))
Load->setMetadata(llvm::LLVMContext::MD_invariant_load,
llvm::MDNode::get(Load->getContext(), {}));
if (isNontemporal) {
diff --git a/clang/test/CodeGenCUDA/invariant-load.cu b/clang/test/CodeGenCUDA/invariant-load.cu
new file mode 100644
index 00000000000000..12685405f9a2a1
--- /dev/null
+++ b/clang/test/CodeGenCUDA/invariant-load.cu
@@ -0,0 +1,43 @@
+// RUN: %clang_cc1 -triple amdgpu7.00-amd-amdhsa -fcuda-is-device -emit-llvm -o - %s | FileCheck %s
+
+#include "Inputs/cuda.h"
+
+__constant__ int constant_value;
+__constant__ int constant_array[4];
+__device__ int device_value;
+
+struct S {
+ int member;
+};
+
+__constant__ S constant_struct;
+
+__device__ int constant_load() {
+ return constant_value;
+}
+
+// CHECK-LABEL: define{{.*}}@_Z13constant_loadv(
+// CHECK: load i32, ptr addrspacecast (ptr addrspace(4) @constant_value to ptr), align 4, !invariant.load [[INVARIANT:![0-9]+]]
+
+__device__ int constant_array_load(int index) {
+ return constant_array[index];
+}
+
+// CHECK-LABEL: define{{.*}}@_Z19constant_array_loadi(
+// CHECK: load i32, ptr %{{.*}}, align 4, !invariant.load [[INVARIANT]]
+
+__device__ int constant_member_load() {
+ return constant_struct.member;
+}
+
+// CHECK-LABEL: define{{.*}}@_Z20constant_member_loadv(
+// CHECK: load i32, ptr addrspacecast (ptr addrspace(4) @constant_struct to ptr), align 4, !invariant.load [[INVARIANT]]
+
+__device__ int device_load() {
+ return device_value;
+}
+
+// CHECK-LABEL: define{{.*}}@_Z11device_loadv(
+// CHECK: load i32, ptr addrspacecast (ptr addrspace(1) @device_value to ptr), align 4{{$}}
+
+// CHECK: [[INVARIANT]] = !{}
diff --git a/clang/test/CodeGenOpenCL/invariant-load.cl b/clang/test/CodeGenOpenCL/invariant-load.cl
index 717348398e28e6..df26f6c4aa495d 100644
--- a/clang/test/CodeGenOpenCL/invariant-load.cl
+++ b/clang/test/CodeGenOpenCL/invariant-load.cl
@@ -1,24 +1,17 @@
-// REQUIRES: amdgpu-registered-target
-// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -cl-std=CL2.0 -O0 -emit-llvm -o - %s | FileCheck %s --check-prefix=AMDGCN
-// RUN: %clang_cc1 -triple spir64-unknown-unknown -cl-std=CL2.0 -O0 -emit-llvm -o - %s | FileCheck %s --check-prefix=SPIR
+// RUN: %clang_cc1 -triple amdgpu7.00-amd-amdhsa -cl-std=CL2.0 -O0 -emit-llvm -o - %s | FileCheck %s
kernel void constant_load(global int *out, constant int *in) {
out[0] = in[0];
}
-// AMDGCN-LABEL: define{{.*}}@constant_load(
-// AMDGCN: load i32, ptr addrspace(4) %{{.*}}, align 4, !invariant.load [[INVARIANT:![0-9]+]]
-// SPIR-LABEL: define{{.*}}@constant_load(
-// SPIR: load i32, ptr addrspace(2) %{{.*}}, align 4, !invariant.load [[INVARIANT:![0-9]+]]
+// CHECK-LABEL: define{{.*}}@constant_load(
+// CHECK: load i32, ptr addrspace(4) %{{.*}}, align 4, !invariant.load [[INVARIANT:![0-9]+]]
kernel void global_const_load(global int *out, global const int *in) {
out[0] = in[0];
}
-// AMDGCN-LABEL: define{{.*}}@global_const_load(
-// AMDGCN: load i32, ptr addrspace(1) %{{.*}}, align 4{{$}}
-// SPIR-LABEL: define{{.*}}@global_const_load(
-// SPIR: load i32, ptr addrspace(1) %{{.*}}, align 4{{$}}
+// CHECK-LABEL: define{{.*}}@global_const_load(
+// CHECK: load i32, ptr addrspace(1) %{{.*}}, align 4{{$}}
-// AMDGCN: [[INVARIANT]] = !{}
-// SPIR: [[INVARIANT]] = !{}
+// CHECK: [[INVARIANT]] = !{}
>From b856f1a40188b4a5d2f472bd2068d964f6ec7bfe Mon Sep 17 00:00:00 2001
From: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
Date: Thu, 10 Sep 2026 10:55:48 +0000
Subject: [PATCH 3/7] [Clang][CUDA] Track invariant loads from AST
Record CUDA constant storage on LValueBaseInfo while the VarDecl is available, and propagate it through fields and conditional lvalues. This avoids recovering pointer provenance from LLVM IR.
Co-authored-by: GPT-5 <noreply at openai.com>
Signed-off-by: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
---
clang/lib/CodeGen/CGExpr.cpp | 42 ++++++++++--------------
clang/lib/CodeGen/CGValue.h | 9 ++++-
clang/test/CodeGenCUDA/invariant-load.cu | 24 ++++++++++++++
3 files changed, 49 insertions(+), 26 deletions(-)
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 845db06a4a0ddc..c49e5cf608e51f 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -42,7 +42,6 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/StringExtras.h"
-#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Intrinsics.h"
@@ -2075,22 +2074,6 @@ llvm::Value *CodeGenFunction::emitScalarConstant(
return Constant.getValue();
}
-static bool isInvariantLoad(CodeGenFunction &CGF, LangAS TypeAS,
- llvm::Value *Ptr) {
- if (TypeAS == LangAS::opencl_constant)
- return true;
-
- // CUDA represents constant memory as a declaration attribute, so the
- // expression type uses the default address space. Recover the storage
- // address space from the underlying global after the address-space cast.
- if (!CGF.getLangOpts().CUDAIsDevice)
- return false;
- const auto *GV =
- llvm::dyn_cast<llvm::GlobalValue>(llvm::getUnderlyingObject(Ptr));
- return GV && GV->getAddressSpace() == CGF.getContext().getTargetAddressSpace(
- LangAS::cuda_constant);
-}
-
llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
SourceLocation Loc) {
return EmitLoadOfScalar(lvalue.getAddress(), lvalue.isVolatile(),
@@ -2265,7 +2248,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
Addr.withElementType(convertTypeForLoadStore(Ty, Addr.getElementType()));
llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile);
- if (isInvariantLoad(*this, Ty.getAddressSpace(), Addr.getBasePointer()))
+ if (Ty.getAddressSpace() == LangAS::opencl_constant || BaseInfo.isInvariant())
Load->setMetadata(llvm::LLVMContext::MD_invariant_load,
llvm::MDNode::get(Load->getContext(), {}));
if (isNontemporal) {
@@ -3520,10 +3503,14 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
E->getExprLoc());
}
- LValue LV = VD->getType()->isReferenceType() ?
- CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
- AlignmentSource::Decl) :
- CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
+ const bool IsReference = VD->getType()->isReferenceType();
+ LValue LV = IsReference ? CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
+ AlignmentSource::Decl)
+ : CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
+ // Preserve CUDA constant storage information from the AST declaration.
+ if (!IsReference && CGF.getLangOpts().CUDAIsDevice &&
+ CGF.CGM.GetGlobalVarAddressSpace(VD) == LangAS::cuda_constant)
+ LV.setInvariant(true);
setObjCGCLValueClass(CGF.getContext(), E, LV);
return LV;
}
@@ -5875,7 +5862,9 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, const FieldDecl *field,
QualType FieldType = field->getType();
const RecordDecl *rec = field->getParent();
AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
- LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(BaseAlignSource));
+ // A field inherits invariant storage from its base object.
+ LValueBaseInfo FieldBaseInfo = BaseInfo;
+ FieldBaseInfo.setAlignmentSource(getFieldAlignmentSource(BaseAlignSource));
TBAAAccessInfo FieldTBAAInfo;
if (base.getTBAAInfo().isMayAlias() ||
rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
@@ -6191,8 +6180,11 @@ LValue CodeGenFunction::EmitConditionalOperatorLValue(
Info.RHS->getBaseInfo().getAlignmentSource());
TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(
Info.LHS->getTBAAInfo(), Info.RHS->getTBAAInfo());
- return MakeAddrLValue(result, expr->getType(), LValueBaseInfo(alignSource),
- TBAAInfo);
+ LValueBaseInfo BaseInfo(alignSource);
+ // Both possible storage locations must be invariant.
+ BaseInfo.setInvariant(Info.LHS->getBaseInfo().isInvariant() &&
+ Info.RHS->getBaseInfo().isInvariant());
+ return MakeAddrLValue(result, expr->getType(), BaseInfo, TBAAInfo);
} else {
assert((Info.LHS || Info.RHS) &&
"both operands of glvalue conditional are throw-expressions?");
diff --git a/clang/lib/CodeGen/CGValue.h b/clang/lib/CodeGen/CGValue.h
index 118ed6690627c0..6eb9ba182fd3e6 100644
--- a/clang/lib/CodeGen/CGValue.h
+++ b/clang/lib/CodeGen/CGValue.h
@@ -166,12 +166,18 @@ static inline AlignmentSource getFieldAlignmentSource(AlignmentSource Source) {
class LValueBaseInfo {
AlignmentSource AlignSource;
+ // Whether loads from the base object's storage are invariant.
+ bool IsInvariant : 1;
+
public:
explicit LValueBaseInfo(AlignmentSource Source = AlignmentSource::Type)
- : AlignSource(Source) {}
+ : AlignSource(Source), IsInvariant(false) {}
AlignmentSource getAlignmentSource() const { return AlignSource; }
void setAlignmentSource(AlignmentSource Source) { AlignSource = Source; }
+ bool isInvariant() const { return IsInvariant; }
+ void setInvariant(bool Value) { IsInvariant = Value; }
+
void mergeForCast(const LValueBaseInfo &Info) {
setAlignmentSource(Info.getAlignmentSource());
}
@@ -357,6 +363,7 @@ class LValue {
LValueBaseInfo getBaseInfo() const { return BaseInfo; }
void setBaseInfo(LValueBaseInfo Info) { BaseInfo = Info; }
+ void setInvariant(bool Value) { BaseInfo.setInvariant(Value); }
KnownNonNull_t isKnownNonNull() const { return Addr.isKnownNonNull(); }
LValue setKnownNonNull() {
diff --git a/clang/test/CodeGenCUDA/invariant-load.cu b/clang/test/CodeGenCUDA/invariant-load.cu
index 12685405f9a2a1..308eb94615ea14 100644
--- a/clang/test/CodeGenCUDA/invariant-load.cu
+++ b/clang/test/CodeGenCUDA/invariant-load.cu
@@ -3,7 +3,9 @@
#include "Inputs/cuda.h"
__constant__ int constant_value;
+__constant__ int other_constant_value;
__constant__ int constant_array[4];
+__constant__ int *constant_pointer;
__device__ int device_value;
struct S {
@@ -40,4 +42,26 @@ __device__ int device_load() {
// CHECK-LABEL: define{{.*}}@_Z11device_loadv(
// CHECK: load i32, ptr addrspacecast (ptr addrspace(1) @device_value to ptr), align 4{{$}}
+__device__ int constant_pointer_load() {
+ return *constant_pointer;
+}
+
+// CHECK-LABEL: define{{.*}}@_Z21constant_pointer_loadv(
+// CHECK: load ptr, ptr addrspacecast (ptr addrspace(4) @constant_pointer to ptr), align 8, !invariant.load [[INVARIANT]]
+// CHECK-NEXT: load i32, ptr %{{.*}}, align 4{{$}}
+
+__device__ int mixed_conditional_load(bool select_constant) {
+ return *&(select_constant ? constant_value : device_value);
+}
+
+// CHECK-LABEL: define{{.*}}@_Z22mixed_conditional_loadb(
+// CHECK: load i32, ptr %{{.*}}, align 4{{$}}
+
+__device__ int constant_conditional_load(bool select_first) {
+ return *&(select_first ? constant_value : other_constant_value);
+}
+
+// CHECK-LABEL: define{{.*}}@_Z25constant_conditional_loadb(
+// CHECK: load i32, ptr %{{.*}}, align 4, !invariant.load [[INVARIANT]]
+
// CHECK: [[INVARIANT]] = !{}
>From 499725e346bcc68fa4c189afe1cf2d8484c3bda8 Mon Sep 17 00:00:00 2001
From: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
Date: Tue, 15 Sep 2026 07:01:34 +0000
Subject: [PATCH 4/7] [Clang][CodeGen] Track invariant state on Address
Move invariant storage tracking from LValueBaseInfo to Address, centralize address-space classification, and preserve the property for the direct array and non-reference field paths covered by the regression tests.
Co-authored-by: GPT-5 <noreply at openai.com>
Signed-off-by: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
---
clang/lib/CodeGen/Address.h | 27 ++++++++++++++++++++----
clang/lib/CodeGen/CGExpr.cpp | 25 +++++++++++-----------
clang/lib/CodeGen/CGValue.h | 15 ++++++-------
clang/lib/CodeGen/CodeGenFunction.h | 2 ++
clang/lib/CodeGen/CodeGenModule.cpp | 4 ++++
clang/lib/CodeGen/CodeGenModule.h | 8 +++++++
clang/test/CodeGenCUDA/invariant-load.cu | 24 ---------------------
7 files changed, 56 insertions(+), 49 deletions(-)
diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h
index 4e7f3561ac0493..a3abd4f47d30e0 100644
--- a/clang/lib/CodeGen/Address.h
+++ b/clang/lib/CodeGen/Address.h
@@ -32,6 +32,9 @@ class CodeGenModule;
// Indicates whether a pointer is known not to be null.
enum KnownNonNull_t { NotKnownNonNull, KnownNonNull };
+// Indicates whether loads from an address are invariant.
+enum KnownInvariant_t { NotKnownInvariant, KnownInvariant };
+
/// An abstract representation of an aligned address. This is designed to be an
/// IR-level abstraction, carrying just the information necessary to perform IR
/// operations on an address like loads and stores. In particular, it doesn't
@@ -145,16 +148,19 @@ class Address {
/// pointer is signed.
llvm::Value *Offset = nullptr;
+ KnownInvariant_t IsInvariant : 1;
+
llvm::Value *emitRawPointerSlow(CodeGenFunction &CGF) const;
protected:
- Address(std::nullptr_t) : ElementType(nullptr) {}
+ Address(std::nullptr_t)
+ : ElementType(nullptr), IsInvariant(NotKnownInvariant) {}
public:
Address(llvm::Value *pointer, llvm::Type *elementType, CharUnits alignment,
KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
: Pointer(pointer, IsKnownNonNull), ElementType(elementType),
- Alignment(alignment) {
+ Alignment(alignment), IsInvariant(NotKnownInvariant) {
assert(pointer != nullptr && "Pointer cannot be null");
assert(elementType != nullptr && "Element type cannot be null");
assert(!alignment.isZero() && "Alignment cannot be zero");
@@ -164,14 +170,16 @@ class Address {
CGPointerAuthInfo PtrAuthInfo, llvm::Value *Offset,
KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
: Pointer(BasePtr, IsKnownNonNull), ElementType(ElementType),
- Alignment(Alignment), PtrAuthInfo(PtrAuthInfo), Offset(Offset) {}
+ Alignment(Alignment), PtrAuthInfo(PtrAuthInfo), Offset(Offset),
+ IsInvariant(NotKnownInvariant) {}
Address(RawAddress RawAddr)
: Pointer(RawAddr.isValid() ? RawAddr.getPointer() : nullptr,
RawAddr.isValid() ? RawAddr.isKnownNonNull() : NotKnownNonNull),
ElementType(RawAddr.isValid() ? RawAddr.getElementType() : nullptr),
Alignment(RawAddr.isValid() ? RawAddr.getAlignment()
- : CharUnits::Zero()) {}
+ : CharUnits::Zero()),
+ IsInvariant(NotKnownInvariant) {}
static Address invalid() { return Address(nullptr); }
bool isValid() const { return Pointer.getPointer() != nullptr; }
@@ -241,6 +249,17 @@ class Address {
return *this;
}
+ KnownInvariant_t isInvariant() const {
+ assert(isValid());
+ return IsInvariant;
+ }
+
+ Address setInvariant(KnownInvariant_t Value) {
+ assert(isValid());
+ IsInvariant = Value;
+ return *this;
+ }
+
bool hasOffset() const { return Offset; }
llvm::Value *getOffset() const { return Offset; }
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index c49e5cf608e51f..054f99ca60fd81 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -2244,11 +2244,13 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
return EmitAtomicLoad(AtomicLValue, Loc).getScalarVal();
}
+ // TODO: Preserve invariance in Address::withElementType.
+ const KnownInvariant_t IsInvariant = Addr.isInvariant();
Addr =
Addr.withElementType(convertTypeForLoadStore(Ty, Addr.getElementType()));
llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile);
- if (Ty.getAddressSpace() == LangAS::opencl_constant || BaseInfo.isInvariant())
+ if (IsInvariant)
Load->setMetadata(llvm::LLVMContext::MD_invariant_load,
llvm::MDNode::get(Load->getContext(), {}));
if (isNontemporal) {
@@ -3507,10 +3509,8 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
LValue LV = IsReference ? CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
AlignmentSource::Decl)
: CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
- // Preserve CUDA constant storage information from the AST declaration.
- if (!IsReference && CGF.getLangOpts().CUDAIsDevice &&
- CGF.CGM.GetGlobalVarAddressSpace(VD) == LangAS::cuda_constant)
- LV.setInvariant(true);
+ if (!IsReference && CGF.CGM.isGlobalVarInvariant(VD))
+ LV.setInvariant(KnownInvariant);
setObjCGCLValueClass(CGF.getContext(), E, LV);
return LV;
}
@@ -5164,6 +5164,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
*this, ArrayLV.getAddress(), {CGM.getSize(CharUnits::Zero()), Idx},
E->getType(), !getLangOpts().PointerOverflowDefined, SignedIndices,
E->getExprLoc(), &arrayType, E->getBase());
+ Addr.setInvariant(ArrayLV.isInvariant());
EltBaseInfo = ArrayLV.getBaseInfo();
if (!CGM.getCodeGenOpts().NewStructPathTBAA) {
// Since CodeGenTBAA::getTypeInfoHelper only handles array types for
@@ -5862,9 +5863,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, const FieldDecl *field,
QualType FieldType = field->getType();
const RecordDecl *rec = field->getParent();
AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
- // A field inherits invariant storage from its base object.
- LValueBaseInfo FieldBaseInfo = BaseInfo;
- FieldBaseInfo.setAlignmentSource(getFieldAlignmentSource(BaseAlignSource));
+ LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(BaseAlignSource));
TBAAAccessInfo FieldTBAAInfo;
if (base.getTBAAInfo().isMayAlias() ||
rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
@@ -5965,6 +5964,9 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, const FieldDecl *field,
if (field->hasAttr<AnnotateAttr>())
addr = EmitFieldAnnotations(field, addr);
+ // Do not propagate invariance to a reference's pointee.
+ if (!field->getType()->isReferenceType())
+ addr.setInvariant(base.isInvariant());
LValue LV = MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);
LV.getQuals().addCVRQualifiers(RecordCVR);
@@ -6180,11 +6182,8 @@ LValue CodeGenFunction::EmitConditionalOperatorLValue(
Info.RHS->getBaseInfo().getAlignmentSource());
TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(
Info.LHS->getTBAAInfo(), Info.RHS->getTBAAInfo());
- LValueBaseInfo BaseInfo(alignSource);
- // Both possible storage locations must be invariant.
- BaseInfo.setInvariant(Info.LHS->getBaseInfo().isInvariant() &&
- Info.RHS->getBaseInfo().isInvariant());
- return MakeAddrLValue(result, expr->getType(), BaseInfo, TBAAInfo);
+ return MakeAddrLValue(result, expr->getType(), LValueBaseInfo(alignSource),
+ TBAAInfo);
} else {
assert((Info.LHS || Info.RHS) &&
"both operands of glvalue conditional are throw-expressions?");
diff --git a/clang/lib/CodeGen/CGValue.h b/clang/lib/CodeGen/CGValue.h
index 6eb9ba182fd3e6..006301a248c2e7 100644
--- a/clang/lib/CodeGen/CGValue.h
+++ b/clang/lib/CodeGen/CGValue.h
@@ -166,18 +166,12 @@ static inline AlignmentSource getFieldAlignmentSource(AlignmentSource Source) {
class LValueBaseInfo {
AlignmentSource AlignSource;
- // Whether loads from the base object's storage are invariant.
- bool IsInvariant : 1;
-
public:
explicit LValueBaseInfo(AlignmentSource Source = AlignmentSource::Type)
- : AlignSource(Source), IsInvariant(false) {}
+ : AlignSource(Source) {}
AlignmentSource getAlignmentSource() const { return AlignSource; }
void setAlignmentSource(AlignmentSource Source) { AlignSource = Source; }
- bool isInvariant() const { return IsInvariant; }
- void setInvariant(bool Value) { IsInvariant = Value; }
-
void mergeForCast(const LValueBaseInfo &Info) {
setAlignmentSource(Info.getAlignmentSource());
}
@@ -363,7 +357,12 @@ class LValue {
LValueBaseInfo getBaseInfo() const { return BaseInfo; }
void setBaseInfo(LValueBaseInfo Info) { BaseInfo = Info; }
- void setInvariant(bool Value) { BaseInfo.setInvariant(Value); }
+
+ KnownInvariant_t isInvariant() const { return Addr.isInvariant(); }
+ LValue setInvariant(KnownInvariant_t Value) {
+ Addr.setInvariant(Value);
+ return *this;
+ }
KnownNonNull_t isKnownNonNull() const { return Addr.isKnownNonNull(); }
LValue setKnownNonNull() {
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index dfb6f2ff65a7d2..a30769ad57aca6 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -2769,6 +2769,8 @@ class CodeGenFunction : public CodeGenTypeCache {
LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,
TBAAAccessInfo TBAAInfo) {
+ if (CGM.isInvariantAddressSpace(T.getAddressSpace()))
+ Addr.setInvariant(KnownInvariant);
return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo);
}
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 435f67542268a8..d4160ad6f81c3f 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -6355,6 +6355,10 @@ LangAS CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D) {
return getTargetCodeGenInfo().getGlobalVarAddressSpace(*this, D);
}
+bool CodeGenModule::isGlobalVarInvariant(const VarDecl *D) {
+ return isInvariantAddressSpace(GetGlobalVarAddressSpace(D));
+}
+
LangAS CodeGenModule::GetGlobalConstantAddressSpace() const {
// OpenCL v1.2 s6.5.3: a string literal is in the constant address space.
if (LangOpts.OpenCL)
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index 1f5ecf734c5288..4cacbd8441fdcf 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -1085,6 +1085,14 @@ class CodeGenModule : public CodeGenTypeCache {
/// space, target-specific global or constant address space may be returned.
LangAS GetGlobalVarAddressSpace(const VarDecl *D);
+ /// Return whether loads from an address space are invariant.
+ bool isInvariantAddressSpace(LangAS AS) const {
+ return AS == LangAS::opencl_constant || AS == LangAS::cuda_constant;
+ }
+
+ /// Return whether loads from a global variable are invariant.
+ bool isGlobalVarInvariant(const VarDecl *D);
+
/// Return the AST address space of constant literal, which is used to emit
/// the constant literal as global variable in LLVM IR.
/// Note: This is not necessarily the address space of the constant literal
diff --git a/clang/test/CodeGenCUDA/invariant-load.cu b/clang/test/CodeGenCUDA/invariant-load.cu
index 308eb94615ea14..12685405f9a2a1 100644
--- a/clang/test/CodeGenCUDA/invariant-load.cu
+++ b/clang/test/CodeGenCUDA/invariant-load.cu
@@ -3,9 +3,7 @@
#include "Inputs/cuda.h"
__constant__ int constant_value;
-__constant__ int other_constant_value;
__constant__ int constant_array[4];
-__constant__ int *constant_pointer;
__device__ int device_value;
struct S {
@@ -42,26 +40,4 @@ __device__ int device_load() {
// CHECK-LABEL: define{{.*}}@_Z11device_loadv(
// CHECK: load i32, ptr addrspacecast (ptr addrspace(1) @device_value to ptr), align 4{{$}}
-__device__ int constant_pointer_load() {
- return *constant_pointer;
-}
-
-// CHECK-LABEL: define{{.*}}@_Z21constant_pointer_loadv(
-// CHECK: load ptr, ptr addrspacecast (ptr addrspace(4) @constant_pointer to ptr), align 8, !invariant.load [[INVARIANT]]
-// CHECK-NEXT: load i32, ptr %{{.*}}, align 4{{$}}
-
-__device__ int mixed_conditional_load(bool select_constant) {
- return *&(select_constant ? constant_value : device_value);
-}
-
-// CHECK-LABEL: define{{.*}}@_Z22mixed_conditional_loadb(
-// CHECK: load i32, ptr %{{.*}}, align 4{{$}}
-
-__device__ int constant_conditional_load(bool select_first) {
- return *&(select_first ? constant_value : other_constant_value);
-}
-
-// CHECK-LABEL: define{{.*}}@_Z25constant_conditional_loadb(
-// CHECK: load i32, ptr %{{.*}}, align 4, !invariant.load [[INVARIANT]]
-
// CHECK: [[INVARIANT]] = !{}
>From a0a8ccae8a681352ba597cd2750e544efc840eb4 Mon Sep 17 00:00:00 2001
From: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
Date: Fri, 6 Nov 2026 07:53:03 +0000
Subject: [PATCH 5/7] Added KnownInvariant as a flag in ElementType; Addressed
other comments.
Signed-off-by: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
---
clang/lib/CodeGen/Address.h | 33 +++-----
clang/lib/CodeGen/CGBuilder.h | 99 +++++++++++++---------
clang/lib/CodeGen/CGExpr.cpp | 20 ++---
clang/lib/CodeGen/CGExprScalar.cpp | 9 +-
clang/lib/CodeGen/CGPointerAuth.cpp | 2 +-
clang/lib/CodeGen/CodeGenFunction.h | 3 +
clang/lib/CodeGen/CodeGenModule.cpp | 4 +
clang/test/CodeGenCUDA/invariant-load.cu | 65 ++++++++++++++
clang/test/CodeGenOpenCL/invariant-load.cl | 13 +++
9 files changed, 168 insertions(+), 80 deletions(-)
diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h
index a3abd4f47d30e0..1b1e6000999ea6 100644
--- a/clang/lib/CodeGen/Address.h
+++ b/clang/lib/CodeGen/Address.h
@@ -137,7 +137,8 @@ class Address {
/// The expected IR type of the pointer. Carrying accurate element type
/// information in Address makes it more convenient to work with Address
/// values and allows frontend assertions to catch simple mistakes.
- llvm::Type *ElementType = nullptr;
+ /// The flag indicates whether loads from the address are invariant.
+ llvm::PointerIntPair<llvm::Type *, 1, KnownInvariant_t> ElementType;
CharUnits Alignment;
@@ -148,19 +149,16 @@ class Address {
/// pointer is signed.
llvm::Value *Offset = nullptr;
- KnownInvariant_t IsInvariant : 1;
-
llvm::Value *emitRawPointerSlow(CodeGenFunction &CGF) const;
protected:
- Address(std::nullptr_t)
- : ElementType(nullptr), IsInvariant(NotKnownInvariant) {}
+ Address(std::nullptr_t) : ElementType(nullptr) {}
public:
Address(llvm::Value *pointer, llvm::Type *elementType, CharUnits alignment,
KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
: Pointer(pointer, IsKnownNonNull), ElementType(elementType),
- Alignment(alignment), IsInvariant(NotKnownInvariant) {
+ Alignment(alignment) {
assert(pointer != nullptr && "Pointer cannot be null");
assert(elementType != nullptr && "Element type cannot be null");
assert(!alignment.isZero() && "Alignment cannot be zero");
@@ -170,16 +168,14 @@ class Address {
CGPointerAuthInfo PtrAuthInfo, llvm::Value *Offset,
KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
: Pointer(BasePtr, IsKnownNonNull), ElementType(ElementType),
- Alignment(Alignment), PtrAuthInfo(PtrAuthInfo), Offset(Offset),
- IsInvariant(NotKnownInvariant) {}
+ Alignment(Alignment), PtrAuthInfo(PtrAuthInfo), Offset(Offset) {}
Address(RawAddress RawAddr)
: Pointer(RawAddr.isValid() ? RawAddr.getPointer() : nullptr,
RawAddr.isValid() ? RawAddr.isKnownNonNull() : NotKnownNonNull),
ElementType(RawAddr.isValid() ? RawAddr.getElementType() : nullptr),
Alignment(RawAddr.isValid() ? RawAddr.getAlignment()
- : CharUnits::Zero()),
- IsInvariant(NotKnownInvariant) {}
+ : CharUnits::Zero()) {}
static Address invalid() { return Address(nullptr); }
bool isValid() const { return Pointer.getPointer() != nullptr; }
@@ -216,7 +212,7 @@ class Address {
/// Return the type of the values stored in this address.
llvm::Type *getElementType() const {
assert(isValid());
- return ElementType;
+ return ElementType.getPointer();
}
/// Return the address space that this address resides in.
@@ -232,7 +228,7 @@ class Address {
void setElementType(llvm::Type *Ty) {
assert(hasOffset() &&
"this funcion shouldn't be called when there is no offset");
- ElementType = Ty;
+ ElementType.setPointer(Ty);
}
bool isSigned() const { return PtrAuthInfo.isSigned(); }
@@ -251,12 +247,12 @@ class Address {
KnownInvariant_t isInvariant() const {
assert(isValid());
- return IsInvariant;
+ return ElementType.getInt();
}
Address setInvariant(KnownInvariant_t Value) {
assert(isValid());
- IsInvariant = Value;
+ ElementType.setInt(Value);
return *this;
}
@@ -287,18 +283,15 @@ class Address {
/// type.
Address withAlignment(CharUnits NewAlignment) const {
return Address(Pointer.getPointer(), getElementType(), NewAlignment,
- isKnownNonNull());
+ isKnownNonNull())
+ .setInvariant(isInvariant());
}
/// Return address with different element type, but same pointer and
/// alignment.
Address withElementType(llvm::Type *ElemTy) const {
- if (!hasOffset())
- return Address(getBasePointer(), ElemTy, getAlignment(),
- getPointerAuthInfo(), /*Offset=*/nullptr,
- isKnownNonNull());
Address A(*this);
- A.ElementType = ElemTy;
+ A.ElementType.setPointer(ElemTy);
return A;
}
};
diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h
index 4a51f6be8578fc..1c1468fede3a50 100644
--- a/clang/lib/CodeGen/CGBuilder.h
+++ b/clang/lib/CodeGen/CGBuilder.h
@@ -87,7 +87,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
return Address(V, ElementTy,
Addr.getAlignment().alignmentAtOffset(
CharUnits::fromQuantity(Offset.getSExtValue())),
- IsInBounds ? Addr.isKnownNonNull() : NotKnownNonNull);
+ IsInBounds ? Addr.isKnownNonNull() : NotKnownNonNull)
+ .setInvariant(Addr.isInvariant());
}
public:
@@ -202,11 +203,13 @@ class CGBuilderTy : public CGBuilderBaseTy {
if (!Addr.hasOffset())
return Address(CreateAddrSpaceCast(Addr.getBasePointer(), Ty, Name),
ElementTy, Addr.getAlignment(), Addr.getPointerAuthInfo(),
- /*Offset=*/nullptr, Addr.isKnownNonNull());
+ /*Offset=*/nullptr, Addr.isKnownNonNull())
+ .setInvariant(Addr.isInvariant());
// Eagerly force a raw address if these is an offset.
- return RawAddress(
- CreateAddrSpaceCast(Addr.emitRawPointer(*getCGF()), Ty, Name),
- ElementTy, Addr.getAlignment(), Addr.isKnownNonNull());
+ return Address(
+ CreateAddrSpaceCast(Addr.emitRawPointer(*getCGF()), Ty, Name),
+ ElementTy, Addr.getAlignment(), Addr.isKnownNonNull())
+ .setInvariant(Addr.isInvariant());
}
using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast;
@@ -237,7 +240,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
Index, Name),
ElTy->getElementType(Index),
Addr.getAlignment().alignmentAtOffset(Offset),
- Addr.isKnownNonNull());
+ Addr.isKnownNonNull())
+ .setInvariant(Addr.isInvariant());
}
/// Given
@@ -255,12 +259,13 @@ class CGBuilderTy : public CGBuilderBaseTy {
CharUnits EltSize =
CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy->getElementType()));
- return Address(
- CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(),
- {getSize(CharUnits::Zero()), getSize(Index)}, Name),
- ElTy->getElementType(),
- Addr.getAlignment().alignmentAtOffset(Index * EltSize),
- Addr.isKnownNonNull());
+ return Address(CreateInBoundsGEP(
+ Addr.getElementType(), Addr.getBasePointer(),
+ {getSize(CharUnits::Zero()), getSize(Index)}, Name),
+ ElTy->getElementType(),
+ Addr.getAlignment().alignmentAtOffset(Index * EltSize),
+ Addr.isKnownNonNull())
+ .setInvariant(Addr.isInvariant());
}
/// Given
@@ -274,10 +279,11 @@ class CGBuilderTy : public CGBuilderBaseTy {
const llvm::DataLayout &DL = BB->getDataLayout();
CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy));
- return Address(
- CreateInBoundsGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name),
- ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize),
- Addr.isKnownNonNull());
+ return Address(CreateInBoundsGEP(ElTy, Addr.getBasePointer(),
+ getSize(Index), Name),
+ ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize),
+ Addr.isKnownNonNull())
+ .setInvariant(Addr.isInvariant());
}
/// Given
@@ -293,7 +299,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
return Address(CreateGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name),
Addr.getElementType(),
- Addr.getAlignment().alignmentAtOffset(Index * EltSize));
+ Addr.getAlignment().alignmentAtOffset(Index * EltSize))
+ .setInvariant(Addr.isInvariant());
}
/// Create GEP with single dynamic index. The address alignment is reduced
@@ -305,21 +312,24 @@ class CGBuilderTy : public CGBuilderBaseTy {
CharUnits EltSize =
CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType()));
- return Address(
- CreateGEP(Addr.getElementType(), Addr.emitRawPointer(CGF), Index, Name),
- Addr.getElementType(),
- Addr.getAlignment().alignmentOfArrayElement(EltSize));
+ return Address(CreateGEP(Addr.getElementType(), Addr.emitRawPointer(CGF),
+ Index, Name),
+ Addr.getElementType(),
+ Addr.getAlignment().alignmentOfArrayElement(EltSize))
+ .setInvariant(Addr.isInvariant());
}
/// Given a pointer to i8, adjust it by a given constant offset.
Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset,
const llvm::Twine &Name = "") {
assert(Addr.getElementType() == TypeCache.Int8Ty);
- return Address(
- CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(),
- getSize(Offset), Name),
- Addr.getElementType(), Addr.getAlignment().alignmentAtOffset(Offset),
- Addr.isKnownNonNull());
+ return Address(CreateInBoundsGEP(Addr.getElementType(),
+ Addr.getBasePointer(), getSize(Offset),
+ Name),
+ Addr.getElementType(),
+ Addr.getAlignment().alignmentAtOffset(Offset),
+ Addr.isKnownNonNull())
+ .setInvariant(Addr.isInvariant());
}
Address CreateConstByteGEP(Address Addr, CharUnits Offset,
@@ -328,7 +338,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
return Address(CreateGEP(Addr.getElementType(), Addr.getBasePointer(),
getSize(Offset), Name),
Addr.getElementType(),
- Addr.getAlignment().alignmentAtOffset(Offset));
+ Addr.getAlignment().alignmentAtOffset(Offset))
+ .setInvariant(Addr.isInvariant());
}
using CGBuilderBaseTy::CreateConstInBoundsGEP2_32;
@@ -348,18 +359,20 @@ class CGBuilderTy : public CGBuilderBaseTy {
const Twine &Name = "",
llvm::GEPNoWrapFlags NW = llvm::GEPNoWrapFlags::none()) {
llvm::Value *Ptr = emitRawPointerFromAddress(Addr);
- return RawAddress(CreateGEP(Addr.getElementType(), Ptr, IdxList, Name, NW),
- ElementType, Align);
+ return Address(CreateGEP(Addr.getElementType(), Ptr, IdxList, Name, NW),
+ ElementType, Align)
+ .setInvariant(Addr.isInvariant());
}
using CGBuilderBaseTy::CreateInBoundsGEP;
Address CreateInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
llvm::Type *ElementType, CharUnits Align,
const Twine &Name = "") {
- return RawAddress(CreateInBoundsGEP(Addr.getElementType(),
- emitRawPointerFromAddress(Addr),
- IdxList, Name),
- ElementType, Align, Addr.isKnownNonNull());
+ return Address(CreateInBoundsGEP(Addr.getElementType(),
+ emitRawPointerFromAddress(Addr), IdxList,
+ Name),
+ ElementType, Align, Addr.isKnownNonNull())
+ .setInvariant(Addr.isInvariant());
}
using CGBuilderBaseTy::CreateStructuredGEP;
@@ -378,10 +391,11 @@ class CGBuilderTy : public CGBuilderBaseTy {
llvm::Type *ElementType, CharUnits Align,
const Twine &Name = "") {
- return RawAddress(CreateAccessChain(Logical, Addr.getElementType(),
- emitRawPointerFromAddress(Addr),
- IdxList, Name),
- ElementType, Align, Addr.isKnownNonNull());
+ return Address(CreateAccessChain(Logical, Addr.getElementType(),
+ emitRawPointerFromAddress(Addr), IdxList,
+ Name),
+ ElementType, Align, Addr.isKnownNonNull())
+ .setInvariant(Addr.isInvariant());
}
using CGBuilderBaseTy::CreateIsNull;
@@ -450,11 +464,12 @@ class CGBuilderTy : public CGBuilderBaseTy {
const llvm::StructLayout *Layout = DL.getStructLayout(ElTy);
auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index));
- return Address(
- CreatePreserveStructAccessIndex(ElTy, emitRawPointerFromAddress(Addr),
- Index, FieldIndex, DbgInfo),
- ElTy->getElementType(Index),
- Addr.getAlignment().alignmentAtOffset(Offset));
+ return Address(CreatePreserveStructAccessIndex(
+ ElTy, emitRawPointerFromAddress(Addr), Index, FieldIndex,
+ DbgInfo),
+ ElTy->getElementType(Index),
+ Addr.getAlignment().alignmentAtOffset(Offset))
+ .setInvariant(Addr.isInvariant());
}
using CGBuilderBaseTy::CreatePreserveUnionAccessIndex;
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 054f99ca60fd81..f24da8eb4faa9b 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -2244,13 +2244,11 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
return EmitAtomicLoad(AtomicLValue, Loc).getScalarVal();
}
- // TODO: Preserve invariance in Address::withElementType.
- const KnownInvariant_t IsInvariant = Addr.isInvariant();
Addr =
Addr.withElementType(convertTypeForLoadStore(Ty, Addr.getElementType()));
llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile);
- if (IsInvariant)
+ if (Addr.isInvariant())
Load->setMetadata(llvm::LLVMContext::MD_invariant_load,
llvm::MDNode::get(Load->getContext(), {}));
if (isNontemporal) {
@@ -3498,6 +3496,8 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());
CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
Address Addr(V, RealVarTy, Alignment);
+ if (CGF.CGM.isGlobalVarInvariant(VD))
+ Addr.setInvariant(KnownInvariant);
// Emit reference to the private copy of the variable if it is an OpenMP
// threadprivate variable.
if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
@@ -3505,12 +3505,10 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
E->getExprLoc());
}
- const bool IsReference = VD->getType()->isReferenceType();
- LValue LV = IsReference ? CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
- AlignmentSource::Decl)
- : CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
- if (!IsReference && CGF.CGM.isGlobalVarInvariant(VD))
- LV.setInvariant(KnownInvariant);
+ LValue LV = VD->getType()->isReferenceType() ?
+ CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
+ AlignmentSource::Decl) :
+ CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
setObjCGCLValueClass(CGF.getContext(), E, LV);
return LV;
}
@@ -5164,7 +5162,6 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
*this, ArrayLV.getAddress(), {CGM.getSize(CharUnits::Zero()), Idx},
E->getType(), !getLangOpts().PointerOverflowDefined, SignedIndices,
E->getExprLoc(), &arrayType, E->getBase());
- Addr.setInvariant(ArrayLV.isInvariant());
EltBaseInfo = ArrayLV.getBaseInfo();
if (!CGM.getCodeGenOpts().NewStructPathTBAA) {
// Since CodeGenTBAA::getTypeInfoHelper only handles array types for
@@ -5964,9 +5961,6 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, const FieldDecl *field,
if (field->hasAttr<AnnotateAttr>())
addr = EmitFieldAnnotations(field, addr);
- // Do not propagate invariance to a reference's pointee.
- if (!field->getType()->isReferenceType())
- addr.setInvariant(base.isInvariant());
LValue LV = MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);
LV.getQuals().addCVRQualifiers(RecordCVR);
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 173931213ce68b..131f6ca58aac4a 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -6561,8 +6561,9 @@ Address CodeGenFunction::EmitCheckedInBoundsGEP(
return Builder.CreateGEP(Addr, IdxList, elementType, Align, Name, NWFlags);
}
- return RawAddress(
- EmitCheckedInBoundsGEP(Addr.getElementType(), Addr.emitRawPointer(*this),
- IdxList, SignedIndices, IsSubtraction, Loc, Name),
- elementType, Align);
+ return Address(EmitCheckedInBoundsGEP(
+ Addr.getElementType(), Addr.emitRawPointer(*this), IdxList,
+ SignedIndices, IsSubtraction, Loc, Name),
+ elementType, Align)
+ .setInvariant(Addr.isInvariant());
}
diff --git a/clang/lib/CodeGen/CGPointerAuth.cpp b/clang/lib/CodeGen/CGPointerAuth.cpp
index 438d08b53a742c..51a4d71f9d6d2b 100644
--- a/clang/lib/CodeGen/CGPointerAuth.cpp
+++ b/clang/lib/CodeGen/CGPointerAuth.cpp
@@ -757,7 +757,7 @@ Address Address::getResignedAddress(const CGPointerAuthInfo &NewInfo,
return Address(getBasePointer(), getElementType(), getAlignment(),
isKnownNonNull());
- assert(ElementType && "Effective type has to be set");
+ assert(getElementType() && "Effective type has to be set");
assert(!Offset && "unexpected non-null offset");
// If the current and the new ptrauth infos are the same and the offset is
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index a30769ad57aca6..01c49e3c2394e2 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -2741,6 +2741,9 @@ class CodeGenFunction : public CodeGenTypeCache {
PtrPhi->addIncoming(RHS.getBasePointer(), RHSBlock);
LHS.replaceBasePointer(PtrPhi);
LHS.setAlignment(std::min(LHS.getAlignment(), RHS.getAlignment()));
+ LHS.setInvariant(LHS.isInvariant() && RHS.isInvariant()
+ ? KnownInvariant
+ : NotKnownInvariant);
return LHS;
}
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index d4160ad6f81c3f..62bc7e63a50888 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -6356,6 +6356,10 @@ LangAS CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D) {
}
bool CodeGenModule::isGlobalVarInvariant(const VarDecl *D) {
+ // Only OpenCL and CUDA device globals are handled here. On other targets,
+ // GetGlobalVarAddressSpace can emit duplicate declaration diagnostics.
+ if (!LangOpts.OpenCL && !(LangOpts.CUDA && LangOpts.CUDAIsDevice))
+ return false;
return isInvariantAddressSpace(GetGlobalVarAddressSpace(D));
}
diff --git a/clang/test/CodeGenCUDA/invariant-load.cu b/clang/test/CodeGenCUDA/invariant-load.cu
index 12685405f9a2a1..994c384eb2b7e8 100644
--- a/clang/test/CodeGenCUDA/invariant-load.cu
+++ b/clang/test/CodeGenCUDA/invariant-load.cu
@@ -1,4 +1,6 @@
// RUN: %clang_cc1 -triple amdgpu7.00-amd-amdhsa -fcuda-is-device -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple amdgpu7.00-amd-amdhsa -fcuda-is-device -fsanitize=array-bounds,pointer-overflow -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple amdgpu7.00-amd-amdhsa -fcuda-is-device -fwrapv-pointer -emit-llvm -o - %s | FileCheck %s
#include "Inputs/cuda.h"
@@ -11,6 +13,7 @@ struct S {
};
__constant__ S constant_struct;
+__device__ S device_struct;
__device__ int constant_load() {
return constant_value;
@@ -40,4 +43,66 @@ __device__ int device_load() {
// CHECK-LABEL: define{{.*}}@_Z11device_loadv(
// CHECK: load i32, ptr addrspacecast (ptr addrspace(1) @device_value to ptr), align 4{{$}}
+struct Inner {
+ int x;
+ int a[4];
+};
+
+struct Outer {
+ Inner in[2];
+ int *p;
+ int &r;
+};
+
+__constant__ Inner constant_inner_array[4];
+__constant__ Outer constant_outer = {{}, &device_value, device_value};
+
+__device__ int nested_array_member_load(int i, int j) {
+ return constant_inner_array[i].a[j];
+}
+
+// CHECK-LABEL: define{{.*}}@_Z24nested_array_member_loadii(
+// CHECK: load i32, ptr %arrayidx{{[0-9]*}}, align 4, !invariant.load [[INVARIANT]]
+
+__device__ int addressed_member_load(int i, int j) {
+ return (&constant_outer)->in[i].a[j];
+}
+
+// CHECK-LABEL: define{{.*}}@_Z21addressed_member_loadii(
+// CHECK: load i32, ptr %arrayidx{{[0-9]*}}, align 4, !invariant.load [[INVARIANT]]
+
+extern __constant__ int &constant_reference;
+
+__device__ int reference_load() {
+ return constant_reference;
+}
+
+// CHECK-LABEL: define{{.*}}@_Z14reference_loadv(
+// CHECK: [[REF:%.*]] = load ptr, ptr addrspacecast (ptr addrspace(4) @constant_reference to ptr)
+// CHECK: load i32, ptr [[REF]], align 4{{$}}
+
+__device__ int pointer_member_index(int i) {
+ return constant_outer.p[i];
+}
+
+// CHECK-LABEL: define{{.*}}@_Z20pointer_member_indexi(
+// CHECK: load ptr, ptr getelementptr {{.*}}@constant_outer{{.*}}, align 8, !invariant.load [[INVARIANT]]
+// CHECK: load i32, ptr %arrayidx, align 4{{$}}
+
+__device__ int reference_member() {
+ return constant_outer.r;
+}
+
+// CHECK-LABEL: define{{.*}}@_Z16reference_memberv(
+// CHECK: [[R:%.*]] = load ptr, ptr getelementptr {{.*}}@constant_outer
+// CHECK: load i32, ptr [[R]], align 4{{$}}
+
+__device__ int conditional_member_load(bool cond) {
+ return (cond ? constant_struct : device_struct).member;
+}
+
+// CHECK-LABEL: define{{.*}}@_Z23conditional_member_loadb(
+// CHECK: phi ptr
+// CHECK: load i32, ptr %{{.*}}, align 4{{$}}
+
// CHECK: [[INVARIANT]] = !{}
diff --git a/clang/test/CodeGenOpenCL/invariant-load.cl b/clang/test/CodeGenOpenCL/invariant-load.cl
index df26f6c4aa495d..c4275912ec1f8f 100644
--- a/clang/test/CodeGenOpenCL/invariant-load.cl
+++ b/clang/test/CodeGenOpenCL/invariant-load.cl
@@ -14,4 +14,17 @@ kernel void global_const_load(global int *out, global const int *in) {
// CHECK-LABEL: define{{.*}}@global_const_load(
// CHECK: load i32, ptr addrspace(1) %{{.*}}, align 4{{$}}
+struct S {
+ int x;
+ int a[4];
+};
+
+kernel void constant_member_array_load(global int *out, constant struct S *in,
+ int i) {
+ out[0] = in->a[i];
+}
+
+// CHECK-LABEL: define{{.*}}@constant_member_array_load(
+// CHECK: load i32, ptr addrspace(4) %arrayidx, align 4, !invariant.load [[INVARIANT]]
+
// CHECK: [[INVARIANT]] = !{}
>From 83a281d899672c5904bd3ea3d85a0ee9f4f2a7f7 Mon Sep 17 00:00:00 2001
From: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
Date: Mon, 9 Nov 2026 03:49:00 +0000
Subject: [PATCH 6/7] [Clang][CodeGen] Classify constant storage independently
of language
`isGlobalVarInvariant()` returned false outside OpenCL and CUDA device
mode, so explicit address-space attributes such as `opencl_constant`
were classified differently by language mode. The guard also hid
duplicate AVR diagnostics from the target `getGlobalVarAddressSpace`
hook.
Check the declared type's address space first, and only consult
`GetGlobalVarAddressSpace()` for CUDA/HIP declaration-based constant
storage, which never reaches the target hook. Treat `hlsl_constant` as
invariant, and route `MakeRawAddrLValue()` and
`MakeAddrLValueWithoutTBAA()` through the checked `MakeAddrLValue()`.
Assisted-by: Claude Opus 5.5
Signed-off-by: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
---
clang/lib/CodeGen/CodeGenFunction.h | 7 +++----
clang/lib/CodeGen/CodeGenModule.cpp | 11 ++++++-----
clang/lib/CodeGen/CodeGenModule.h | 3 ++-
clang/test/CodeGenHLSL/invariant-load.hlsl | 15 +++++++++++++++
4 files changed, 26 insertions(+), 10 deletions(-)
create mode 100644 clang/test/CodeGenHLSL/invariant-load.hlsl
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 01c49e3c2394e2..c68048ff76d6db 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -2788,15 +2788,14 @@ class CodeGenFunction : public CodeGenTypeCache {
LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
AlignmentSource Source = AlignmentSource::Type) {
Address Addr(V, ConvertTypeForMem(T), Alignment);
- return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source),
- CGM.getTBAAAccessInfo(T));
+ return MakeAddrLValue(Addr, T, LValueBaseInfo(Source),
+ CGM.getTBAAAccessInfo(T));
}
LValue
MakeAddrLValueWithoutTBAA(Address Addr, QualType T,
AlignmentSource Source = AlignmentSource::Type) {
- return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source),
- TBAAAccessInfo());
+ return MakeAddrLValue(Addr, T, LValueBaseInfo(Source), TBAAAccessInfo());
}
/// Given a value of type T* that may not be to a complete object, construct
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 62bc7e63a50888..a6494e572e9d96 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -6356,11 +6356,12 @@ LangAS CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D) {
}
bool CodeGenModule::isGlobalVarInvariant(const VarDecl *D) {
- // Only OpenCL and CUDA device globals are handled here. On other targets,
- // GetGlobalVarAddressSpace can emit duplicate declaration diagnostics.
- if (!LangOpts.OpenCL && !(LangOpts.CUDA && LangOpts.CUDAIsDevice))
- return false;
- return isInvariantAddressSpace(GetGlobalVarAddressSpace(D));
+ if (isInvariantAddressSpace(D->getType().getAddressSpace()))
+ return true;
+ // CUDA and HIP assign constant storage from declaration attributes, not
+ // from the variable's type.
+ return LangOpts.CUDA && LangOpts.CUDAIsDevice &&
+ isInvariantAddressSpace(GetGlobalVarAddressSpace(D));
}
LangAS CodeGenModule::GetGlobalConstantAddressSpace() const {
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index 4cacbd8441fdcf..cda47d931dd569 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -1087,7 +1087,8 @@ class CodeGenModule : public CodeGenTypeCache {
/// Return whether loads from an address space are invariant.
bool isInvariantAddressSpace(LangAS AS) const {
- return AS == LangAS::opencl_constant || AS == LangAS::cuda_constant;
+ return AS == LangAS::opencl_constant || AS == LangAS::cuda_constant ||
+ AS == LangAS::hlsl_constant;
}
/// Return whether loads from a global variable are invariant.
diff --git a/clang/test/CodeGenHLSL/invariant-load.hlsl b/clang/test/CodeGenHLSL/invariant-load.hlsl
new file mode 100644
index 00000000000000..d8c1fd8c6f7f88
--- /dev/null
+++ b/clang/test/CodeGenHLSL/invariant-load.hlsl
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+
+cbuffer CB {
+ float a;
+ float b[4];
+}
+
+// CHECK-LABEL: define{{.*}}@_Z13cbuffer_loadsi(
+// CHECK: load float, ptr addrspace(2) @a, align 4, !invariant.load [[INVARIANT:![0-9]+]]
+// CHECK: load float, ptr addrspace(2) %cbufferidx, align 16, !invariant.load [[INVARIANT]]
+export float cbuffer_loads(int i) {
+ return a + b[i];
+}
+
+// CHECK: [[INVARIANT]] = !{}
>From f04b3948087b381e83e4ddfaf90b19aa1d0e3cd0 Mon Sep 17 00:00:00 2001
From: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
Date: Mon, 9 Nov 2026 07:32:23 +0000
Subject: [PATCH 7/7] [Clang][CodeGen] Initialize invariance in Address
constructors
`Address` construction required a separate `setInvariant()` call, so
address-producing helpers repeatedly initialized and then updated the
invariant flag.
Add an optional `KnownInvariant_t` argument to both pointer constructors
and pass it at construction sites. Preserve the base flag in
`emitBufferMemberExpr()` so HLSL constant-buffer member loads retain
`!invariant.load`.
Tests: Clang build, 10 targeted lit tests, constructor smoke checks.
Assisted-by: OpenCode
Signed-off-by: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
---
clang/lib/CodeGen/Address.h | 18 ++--
clang/lib/CodeGen/CGBuilder.h | 102 ++++++++++-----------
clang/lib/CodeGen/CGExpr.cpp | 6 +-
clang/lib/CodeGen/CGExprScalar.cpp | 9 +-
clang/lib/CodeGen/CGHLSLRuntime.cpp | 2 +-
clang/test/CodeGenHLSL/invariant-load.hlsl | 12 +++
6 files changed, 77 insertions(+), 72 deletions(-)
diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h
index 1b1e6000999ea6..20bea57fd01b9d 100644
--- a/clang/lib/CodeGen/Address.h
+++ b/clang/lib/CodeGen/Address.h
@@ -156,9 +156,10 @@ class Address {
public:
Address(llvm::Value *pointer, llvm::Type *elementType, CharUnits alignment,
- KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
- : Pointer(pointer, IsKnownNonNull), ElementType(elementType),
- Alignment(alignment) {
+ KnownNonNull_t IsKnownNonNull = NotKnownNonNull,
+ KnownInvariant_t IsKnownInvariant = NotKnownInvariant)
+ : Pointer(pointer, IsKnownNonNull),
+ ElementType(elementType, IsKnownInvariant), Alignment(alignment) {
assert(pointer != nullptr && "Pointer cannot be null");
assert(elementType != nullptr && "Element type cannot be null");
assert(!alignment.isZero() && "Alignment cannot be zero");
@@ -166,9 +167,11 @@ class Address {
Address(llvm::Value *BasePtr, llvm::Type *ElementType, CharUnits Alignment,
CGPointerAuthInfo PtrAuthInfo, llvm::Value *Offset,
- KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
- : Pointer(BasePtr, IsKnownNonNull), ElementType(ElementType),
- Alignment(Alignment), PtrAuthInfo(PtrAuthInfo), Offset(Offset) {}
+ KnownNonNull_t IsKnownNonNull = NotKnownNonNull,
+ KnownInvariant_t IsKnownInvariant = NotKnownInvariant)
+ : Pointer(BasePtr, IsKnownNonNull),
+ ElementType(ElementType, IsKnownInvariant), Alignment(Alignment),
+ PtrAuthInfo(PtrAuthInfo), Offset(Offset) {}
Address(RawAddress RawAddr)
: Pointer(RawAddr.isValid() ? RawAddr.getPointer() : nullptr,
@@ -283,8 +286,7 @@ class Address {
/// type.
Address withAlignment(CharUnits NewAlignment) const {
return Address(Pointer.getPointer(), getElementType(), NewAlignment,
- isKnownNonNull())
- .setInvariant(isInvariant());
+ isKnownNonNull(), isInvariant());
}
/// Return address with different element type, but same pointer and
diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h
index 1c1468fede3a50..0502d819e90a60 100644
--- a/clang/lib/CodeGen/CGBuilder.h
+++ b/clang/lib/CodeGen/CGBuilder.h
@@ -87,8 +87,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
return Address(V, ElementTy,
Addr.getAlignment().alignmentAtOffset(
CharUnits::fromQuantity(Offset.getSExtValue())),
- IsInBounds ? Addr.isKnownNonNull() : NotKnownNonNull)
- .setInvariant(Addr.isInvariant());
+ IsInBounds ? Addr.isKnownNonNull() : NotKnownNonNull,
+ Addr.isInvariant());
}
public:
@@ -203,13 +203,13 @@ class CGBuilderTy : public CGBuilderBaseTy {
if (!Addr.hasOffset())
return Address(CreateAddrSpaceCast(Addr.getBasePointer(), Ty, Name),
ElementTy, Addr.getAlignment(), Addr.getPointerAuthInfo(),
- /*Offset=*/nullptr, Addr.isKnownNonNull())
- .setInvariant(Addr.isInvariant());
+ /*Offset=*/nullptr, Addr.isKnownNonNull(),
+ Addr.isInvariant());
// Eagerly force a raw address if these is an offset.
return Address(
- CreateAddrSpaceCast(Addr.emitRawPointer(*getCGF()), Ty, Name),
- ElementTy, Addr.getAlignment(), Addr.isKnownNonNull())
- .setInvariant(Addr.isInvariant());
+ CreateAddrSpaceCast(Addr.emitRawPointer(*getCGF()), Ty, Name),
+ ElementTy, Addr.getAlignment(), Addr.isKnownNonNull(),
+ Addr.isInvariant());
}
using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast;
@@ -240,8 +240,7 @@ class CGBuilderTy : public CGBuilderBaseTy {
Index, Name),
ElTy->getElementType(Index),
Addr.getAlignment().alignmentAtOffset(Offset),
- Addr.isKnownNonNull())
- .setInvariant(Addr.isInvariant());
+ Addr.isKnownNonNull(), Addr.isInvariant());
}
/// Given
@@ -259,13 +258,12 @@ class CGBuilderTy : public CGBuilderBaseTy {
CharUnits EltSize =
CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy->getElementType()));
- return Address(CreateInBoundsGEP(
- Addr.getElementType(), Addr.getBasePointer(),
- {getSize(CharUnits::Zero()), getSize(Index)}, Name),
- ElTy->getElementType(),
- Addr.getAlignment().alignmentAtOffset(Index * EltSize),
- Addr.isKnownNonNull())
- .setInvariant(Addr.isInvariant());
+ return Address(
+ CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(),
+ {getSize(CharUnits::Zero()), getSize(Index)}, Name),
+ ElTy->getElementType(),
+ Addr.getAlignment().alignmentAtOffset(Index * EltSize),
+ Addr.isKnownNonNull(), Addr.isInvariant());
}
/// Given
@@ -279,11 +277,10 @@ class CGBuilderTy : public CGBuilderBaseTy {
const llvm::DataLayout &DL = BB->getDataLayout();
CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy));
- return Address(CreateInBoundsGEP(ElTy, Addr.getBasePointer(),
- getSize(Index), Name),
- ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize),
- Addr.isKnownNonNull())
- .setInvariant(Addr.isInvariant());
+ return Address(
+ CreateInBoundsGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name),
+ ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize),
+ Addr.isKnownNonNull(), Addr.isInvariant());
}
/// Given
@@ -299,8 +296,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
return Address(CreateGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name),
Addr.getElementType(),
- Addr.getAlignment().alignmentAtOffset(Index * EltSize))
- .setInvariant(Addr.isInvariant());
+ Addr.getAlignment().alignmentAtOffset(Index * EltSize),
+ NotKnownNonNull, Addr.isInvariant());
}
/// Create GEP with single dynamic index. The address alignment is reduced
@@ -312,24 +309,22 @@ class CGBuilderTy : public CGBuilderBaseTy {
CharUnits EltSize =
CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType()));
- return Address(CreateGEP(Addr.getElementType(), Addr.emitRawPointer(CGF),
- Index, Name),
- Addr.getElementType(),
- Addr.getAlignment().alignmentOfArrayElement(EltSize))
- .setInvariant(Addr.isInvariant());
+ return Address(
+ CreateGEP(Addr.getElementType(), Addr.emitRawPointer(CGF), Index, Name),
+ Addr.getElementType(),
+ Addr.getAlignment().alignmentOfArrayElement(EltSize), NotKnownNonNull,
+ Addr.isInvariant());
}
/// Given a pointer to i8, adjust it by a given constant offset.
Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset,
const llvm::Twine &Name = "") {
assert(Addr.getElementType() == TypeCache.Int8Ty);
- return Address(CreateInBoundsGEP(Addr.getElementType(),
- Addr.getBasePointer(), getSize(Offset),
- Name),
- Addr.getElementType(),
- Addr.getAlignment().alignmentAtOffset(Offset),
- Addr.isKnownNonNull())
- .setInvariant(Addr.isInvariant());
+ return Address(
+ CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(),
+ getSize(Offset), Name),
+ Addr.getElementType(), Addr.getAlignment().alignmentAtOffset(Offset),
+ Addr.isKnownNonNull(), Addr.isInvariant());
}
Address CreateConstByteGEP(Address Addr, CharUnits Offset,
@@ -338,8 +333,8 @@ class CGBuilderTy : public CGBuilderBaseTy {
return Address(CreateGEP(Addr.getElementType(), Addr.getBasePointer(),
getSize(Offset), Name),
Addr.getElementType(),
- Addr.getAlignment().alignmentAtOffset(Offset))
- .setInvariant(Addr.isInvariant());
+ Addr.getAlignment().alignmentAtOffset(Offset),
+ NotKnownNonNull, Addr.isInvariant());
}
using CGBuilderBaseTy::CreateConstInBoundsGEP2_32;
@@ -360,19 +355,17 @@ class CGBuilderTy : public CGBuilderBaseTy {
llvm::GEPNoWrapFlags NW = llvm::GEPNoWrapFlags::none()) {
llvm::Value *Ptr = emitRawPointerFromAddress(Addr);
return Address(CreateGEP(Addr.getElementType(), Ptr, IdxList, Name, NW),
- ElementType, Align)
- .setInvariant(Addr.isInvariant());
+ ElementType, Align, NotKnownNonNull, Addr.isInvariant());
}
using CGBuilderBaseTy::CreateInBoundsGEP;
Address CreateInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
llvm::Type *ElementType, CharUnits Align,
const Twine &Name = "") {
- return Address(CreateInBoundsGEP(Addr.getElementType(),
- emitRawPointerFromAddress(Addr), IdxList,
- Name),
- ElementType, Align, Addr.isKnownNonNull())
- .setInvariant(Addr.isInvariant());
+ return Address(
+ CreateInBoundsGEP(Addr.getElementType(),
+ emitRawPointerFromAddress(Addr), IdxList, Name),
+ ElementType, Align, Addr.isKnownNonNull(), Addr.isInvariant());
}
using CGBuilderBaseTy::CreateStructuredGEP;
@@ -391,11 +384,10 @@ class CGBuilderTy : public CGBuilderBaseTy {
llvm::Type *ElementType, CharUnits Align,
const Twine &Name = "") {
- return Address(CreateAccessChain(Logical, Addr.getElementType(),
- emitRawPointerFromAddress(Addr), IdxList,
- Name),
- ElementType, Align, Addr.isKnownNonNull())
- .setInvariant(Addr.isInvariant());
+ return Address(
+ CreateAccessChain(Logical, Addr.getElementType(),
+ emitRawPointerFromAddress(Addr), IdxList, Name),
+ ElementType, Align, Addr.isKnownNonNull(), Addr.isInvariant());
}
using CGBuilderBaseTy::CreateIsNull;
@@ -464,12 +456,12 @@ class CGBuilderTy : public CGBuilderBaseTy {
const llvm::StructLayout *Layout = DL.getStructLayout(ElTy);
auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index));
- return Address(CreatePreserveStructAccessIndex(
- ElTy, emitRawPointerFromAddress(Addr), Index, FieldIndex,
- DbgInfo),
- ElTy->getElementType(Index),
- Addr.getAlignment().alignmentAtOffset(Offset))
- .setInvariant(Addr.isInvariant());
+ return Address(
+ CreatePreserveStructAccessIndex(ElTy, emitRawPointerFromAddress(Addr),
+ Index, FieldIndex, DbgInfo),
+ ElTy->getElementType(Index),
+ Addr.getAlignment().alignmentAtOffset(Offset), NotKnownNonNull,
+ Addr.isInvariant());
}
using CGBuilderBaseTy::CreatePreserveUnionAccessIndex;
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index f24da8eb4faa9b..07e387c8f7c387 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -3495,9 +3495,9 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());
CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
- Address Addr(V, RealVarTy, Alignment);
- if (CGF.CGM.isGlobalVarInvariant(VD))
- Addr.setInvariant(KnownInvariant);
+ Address Addr(V, RealVarTy, Alignment, NotKnownNonNull,
+ CGF.CGM.isGlobalVarInvariant(VD) ? KnownInvariant
+ : NotKnownInvariant);
// Emit reference to the private copy of the variable if it is an OpenMP
// threadprivate variable.
if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 131f6ca58aac4a..0d44d6c2d87d09 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -6561,9 +6561,8 @@ Address CodeGenFunction::EmitCheckedInBoundsGEP(
return Builder.CreateGEP(Addr, IdxList, elementType, Align, Name, NWFlags);
}
- return Address(EmitCheckedInBoundsGEP(
- Addr.getElementType(), Addr.emitRawPointer(*this), IdxList,
- SignedIndices, IsSubtraction, Loc, Name),
- elementType, Align)
- .setInvariant(Addr.isInvariant());
+ return Address(
+ EmitCheckedInBoundsGEP(Addr.getElementType(), Addr.emitRawPointer(*this),
+ IdxList, SignedIndices, IsSubtraction, Loc, Name),
+ elementType, Align, NotKnownNonNull, Addr.isInvariant());
}
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index 05f755edde64b9..4dd36b74c41056 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -2378,7 +2378,7 @@ LValue CGHLSLRuntime::emitBufferMemberExpr(CodeGenFunction &CGF,
llvm::ConstantInt::get(CGM.IntTy, FieldIdx))
: CGF.Builder.CreateStructGEP(LayoutTy, Base.getPointer(CGF),
FieldIdx, Field->getName());
- Address Addr(Ptr, FieldLLVMTy, Align, KnownNonNull);
+ Address Addr(Ptr, FieldLLVMTy, Align, KnownNonNull, Base.isInvariant());
LValue LV = LValue::MakeAddr(Addr, FieldType, CGM.getContext(),
LValueBaseInfo(AlignmentSource::Type),
diff --git a/clang/test/CodeGenHLSL/invariant-load.hlsl b/clang/test/CodeGenHLSL/invariant-load.hlsl
index d8c1fd8c6f7f88..fd9851d996a9ac 100644
--- a/clang/test/CodeGenHLSL/invariant-load.hlsl
+++ b/clang/test/CodeGenHLSL/invariant-load.hlsl
@@ -1,8 +1,14 @@
// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+struct S {
+ float x;
+ float y;
+};
+
cbuffer CB {
float a;
float b[4];
+ S s;
}
// CHECK-LABEL: define{{.*}}@_Z13cbuffer_loadsi(
@@ -12,4 +18,10 @@ export float cbuffer_loads(int i) {
return a + b[i];
}
+// CHECK-LABEL: define{{.*}}@_Z13struct_memberv(
+// CHECK: load float, ptr addrspace(2) getelementptr {{.*}}@s{{.*}}, align 4, !invariant.load [[INVARIANT]]
+export float struct_member() {
+ return s.y;
+}
+
// CHECK: [[INVARIANT]] = !{}
More information about the cfe-commits
mailing list