[llvm] [AArch64][Bitcode] Use target memory for SME state (PR #205829)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 25 07:31:17 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: CarolineConcatto
<details>
<summary>Changes</summary>
Model AArch64 ZA and ZT0 intrinsic state using target_mem instead of inaccessiblemem.
Bump the bitcode memory-attribute encoding and upgrade old AArch64 bitcode so prior inaccessiblemem effects are preserved on the new target memory locations. Non-AArch64 bitcode keeps the old interpretation.
---
Full diff: https://github.com/llvm/llvm-project/pull/205829.diff
8 Files Affected:
- (modified) llvm/docs/LangRef.rst (+1-2)
- (modified) llvm/include/llvm/IR/IntrinsicsAArch64.td (+2-2)
- (modified) llvm/lib/Bitcode/Reader/BitcodeReader.cpp (+29-5)
- (modified) llvm/lib/Bitcode/Writer/BitcodeWriter.cpp (+1-1)
- (added) llvm/test/Bitcode/Inputs/aarch64-memory-attribute-upgrade.bc ()
- (added) llvm/test/Bitcode/Inputs/x86-memory-attribute-upgrade.bc ()
- (added) llvm/test/Bitcode/aarch64-target-memory-attribute.ll (+36)
- (added) llvm/test/Transforms/LICM/AArch64/sme-fp8-hoits.ll (+52)
``````````diff
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 287341d787b64..66dce0077d7de 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -2292,8 +2292,7 @@ For example:
- ``errnomem``: This refers to accesses to the ``errno`` variable.
- ``target_mem#`` : These refer to target specific state that cannot be
accessed by any other means. # is a number between 0 and 1 inclusive.
- Note: The target_mem locations are experimental and intended for internal
- testing only. They must not be used in production code.
+ Note: The target_mem locations are implemented in AArch64.
- The default access kind (specified without a location prefix) applies to
all locations that haven't been specified explicitly, including those that
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 5ba1f4ba861d2..6078e1438ceee 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -734,8 +734,8 @@ def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;
// Maps Memory locations to registers.
defvar FPMR = InaccessibleMem;
-defvar ZT0 = InaccessibleMem;
-defvar ZA = InaccessibleMem;
+defvar ZT0 = TargetMem0;
+defvar ZA = TargetMem1;
let TargetPrefix = "aarch64" in {
class FPENV_Get_Intrinsic
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index c913dd811a2e0..a5f59a795b4ce 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -581,6 +581,7 @@ class BitcodeConstant final : public Value,
class BitcodeReader : public BitcodeReaderBase, public GVMaterializer {
LLVMContext &Context;
Module *TheModule = nullptr;
+ Triple BitcodeTargetTriple;
// Next offset to start scanning for lazy parsing of function bodies.
uint64_t NextUnreadBit = 0;
// Last function offset found in the VST.
@@ -705,7 +706,8 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer {
public:
BitcodeReader(BitstreamCursor Stream, StringRef Strtab,
- StringRef ProducerIdentification, LLVMContext &Context);
+ StringRef ProducerIdentification, LLVMContext &Context,
+ Triple BitcodeTargetTriple);
Error materializeForwardReferencedFunctions();
@@ -1063,8 +1065,9 @@ std::error_code llvm::errorToErrorCodeAndEmitErrors(LLVMContext &Ctx,
BitcodeReader::BitcodeReader(BitstreamCursor Stream, StringRef Strtab,
StringRef ProducerIdentification,
- LLVMContext &Context)
+ LLVMContext &Context, Triple TTriple)
: BitcodeReaderBase(std::move(Stream), Strtab), Context(Context),
+ BitcodeTargetTriple(TTriple),
ValueList(this->Stream.SizeInBytes(),
[this](unsigned ValID, BasicBlock *InsertBB) {
return materializeValue(ValID, InsertBB);
@@ -2469,12 +2472,24 @@ Error BitcodeReader::parseAttributeGroupBlock() {
MemoryEffects::argMemOnly(ArgMem) |
MemoryEffects::errnoMemOnly(OtherMem) |
MemoryEffects::otherMemOnly(OtherMem);
+ // Old versions dont have target memory location.
+ // It was represented as Inaccessible memory for AArch64.
+ if (BitcodeTargetTriple.isAArch64())
+ for (auto Loc : MemoryEffects::targetMemLocations())
+ ME = ME.getWithModRef(Loc, InaccessibleMem);
B.addMemoryAttr(ME);
} else {
// Construct the memory attribute directly from the encoded base
// on newer versions.
- B.addMemoryAttr(MemoryEffects::createFromIntValue(
- EncodedME & 0x00FFFFFFFFFFFFFFULL));
+ auto ME = MemoryEffects::createFromIntValue(
+ EncodedME & 0x00FFFFFFFFFFFFFFULL);
+ // Only from Version=2 onwards target memory location exist.
+ // It was represented as Inaccessible memory for AArch64.
+ if (Version == 1 && BitcodeTargetTriple.isAArch64())
+ for (auto Loc : MemoryEffects::targetMemLocations())
+ ME = ME.getWithModRef(
+ Loc, ME.getModRef(IRMemLocation::InaccessibleMem));
+ B.addMemoryAttr(ME);
}
} else if (Kind == Attribute::Captures)
B.addCapturesAttr(CaptureInfo::createFromIntValue(Record[++i]));
@@ -8700,10 +8715,19 @@ BitcodeModule::getModuleImpl(LLVMContext &Context, bool MaterializeAll,
return std::move(E);
}
+ // Some invalid bitcode files have multiple independent errors.
+ // IF/ELSE avoids printing failure in this preliminary scan.
+ Triple BitcodeTargetTriple;
+ BitstreamCursor TripleStream(Buffer);
+ if (Expected<std::string> TripleStr = readTriple(TripleStream))
+ BitcodeTargetTriple = Triple(*TripleStr);
+ else
+ consumeError(TripleStr.takeError());
+
if (Error JumpFailed = Stream.JumpToBit(ModuleBit))
return std::move(JumpFailed);
auto *R = new BitcodeReader(std::move(Stream), Strtab, ProducerIdentification,
- Context);
+ Context, BitcodeTargetTriple);
std::unique_ptr<Module> M =
std::make_unique<Module>(ModuleIdentifier, Context);
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index f4857461ca58e..29b621eed2700 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -1078,7 +1078,7 @@ void ModuleBitcodeWriter::writeAttributeGroupTable() {
Record.push_back(getAttrKindEncoding(Kind));
if (Kind == Attribute::Memory) {
// Version field for upgrading old memory effects.
- const uint64_t Version = 1;
+ const uint64_t Version = 2;
Record.push_back((Version << 56) | Attr.getValueAsInt());
} else {
Record.push_back(Attr.getValueAsInt());
diff --git a/llvm/test/Bitcode/Inputs/aarch64-memory-attribute-upgrade.bc b/llvm/test/Bitcode/Inputs/aarch64-memory-attribute-upgrade.bc
new file mode 100644
index 0000000000000..26bc0339df518
Binary files /dev/null and b/llvm/test/Bitcode/Inputs/aarch64-memory-attribute-upgrade.bc differ
diff --git a/llvm/test/Bitcode/Inputs/x86-memory-attribute-upgrade.bc b/llvm/test/Bitcode/Inputs/x86-memory-attribute-upgrade.bc
new file mode 100644
index 0000000000000..d40ceb9b2e459
Binary files /dev/null and b/llvm/test/Bitcode/Inputs/x86-memory-attribute-upgrade.bc differ
diff --git a/llvm/test/Bitcode/aarch64-target-memory-attribute.ll b/llvm/test/Bitcode/aarch64-target-memory-attribute.ll
new file mode 100644
index 0000000000000..1ead9e5e878ba
--- /dev/null
+++ b/llvm/test/Bitcode/aarch64-target-memory-attribute.ll
@@ -0,0 +1,36 @@
+; RUN: llvm-dis < %S/Inputs/aarch64-memory-attribute-upgrade.bc | FileCheck %s --check-prefix=AARCH64
+; RUN: llvm-dis < %S/Inputs/x86-memory-attribute-upgrade.bc | FileCheck %s --check-prefix=NON-AARCH64
+
+; The .bc inputs were generated by an older LLVM memory-attribute encoding,
+; before AArch64 target memory was split out from inaccessible memory.
+; The code was compiled using x86 and aarch64 hosts.
+
+target triple = "aarch64-linux"
+
+define void @test_inaccessible_read() #0 {
+; AARCH64: ; Function Attrs: memory(inaccessiblemem: read, target_mem: read)
+; AARCH64-NEXT: define void @test_inaccessible_read()
+; NON-AARCH64: ; Function Attrs: memory(inaccessiblemem: read)
+; NON-AARCH64-NEXT: define void @test_inaccessible_read()
+ ret void
+}
+
+define void @test_inaccessible_readwrite() #1 {
+; AARCH64: ; Function Attrs: memory(inaccessiblemem: readwrite, target_mem: readwrite)
+; AARCH64-NEXT: define void @test_inaccessible_readwrite()
+; NON-AARCH64: ; Function Attrs: memory(inaccessiblemem: readwrite)
+; NON-AARCH64-NEXT: define void @test_inaccessible_readwrite()
+ ret void
+}
+
+define void @test_inaccessible_none() #2 {
+; AARCH64: ; Function Attrs: memory(none)
+; AARCH64-NEXT: define void @test_inaccessible_none()
+; NON-AARCH64: ; Function Attrs: memory(none)
+; NON-AARCH64-NEXT: define void @test_inaccessible_none()
+ ret void
+}
+
+attributes #0 = { memory(inaccessiblemem: read) }
+attributes #1 = { memory(inaccessiblemem: readwrite) }
+attributes #2 = { memory(inaccessiblemem: none) }
diff --git a/llvm/test/Transforms/LICM/AArch64/sme-fp8-hoits.ll b/llvm/test/Transforms/LICM/AArch64/sme-fp8-hoits.ll
new file mode 100644
index 0000000000000..1efb6688e0208
--- /dev/null
+++ b/llvm/test/Transforms/LICM/AArch64/sme-fp8-hoits.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=licm < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;; The set.fprm should be hoisted once ZA does not use the same
+;; memory location as FPMR.
+define void @fp8_fmopa_loop(ptr %lhs, ptr %rhs, i64 %fpmr, i64 %n) {
+; CHECK-LABEL: define void @fp8_fmopa_loop(
+; CHECK-SAME: ptr [[LHS:%.*]], ptr [[RHS:%.*]], i64 [[FPMR:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[PTRUE:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+; CHECK-NEXT: [[LHS_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8.p0(<vscale x 16 x i1> [[PTRUE]], ptr [[LHS]])
+; CHECK-NEXT: [[RHS_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8.p0(<vscale x 16 x i1> [[PTRUE]], ptr [[RHS]])
+; CHECK-NEXT: call void @llvm.aarch64.set.fpmr(i64 [[FPMR]])
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: call void @llvm.aarch64.sme.fp8.fmopa.za32(i32 0, <vscale x 16 x i1> [[PTRUE]], <vscale x 16 x i1> [[PTRUE]], <vscale x 16 x i8> [[LHS_LOAD]], <vscale x 16 x i8> [[RHS_LOAD]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %ptrue = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+ %lhs.load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %ptrue, ptr %lhs)
+ %rhs.load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1> %ptrue, ptr %rhs)
+ call void @llvm.aarch64.set.fpmr(i64 %fpmr)
+ call void @llvm.aarch64.sme.fp8.fmopa.za32(i32 0, <vscale x 16 x i1> %ptrue, <vscale x 16 x i1> %ptrue, <vscale x 16 x i8> %lhs.load, <vscale x 16 x i8> %rhs.load)
+ %iv.next = add nuw i64 %iv, 1
+ %cond = icmp ult i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 immarg) #0
+declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1>, ptr) #1
+declare void @llvm.aarch64.set.fpmr(i64) #2
+declare void @llvm.aarch64.sme.fp8.fmopa.za32(i32 immarg, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>) #3
+
+attributes #0 = { nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
+attributes #2 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+attributes #3 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite) }
``````````
</details>
https://github.com/llvm/llvm-project/pull/205829
More information about the llvm-commits
mailing list