[clang] [CIR] Move records through memory in x86_64 call lowering (PR #231663)
Adam Smith via cfe-commits
cfe-commits at lists.llvm.org
Wed Oct 14 13:46:46 PDT 2026
https://github.com/adams381 updated https://github.com/llvm/llvm-project/pull/231663
>From 0e27435cf8398c3bb0402dd24904d5a7cb7a2f6a Mon Sep 17 00:00:00 2001
From: Adam Smith <adams at nvidia.com>
Date: Tue, 13 Oct 2026 13:39:40 -0700
Subject: [PATCH 1/2] [CIR] Move records through memory in x86_64 call lowering
CIR lowers a union to its storage type, so moving one as an SSA value
can drop bytes. An x86_fp80 member stores only 10 of its 16 bytes, for
example. The calling-convention rewriter now passes records through
memory at every coercion site, the way classic CodeGen does. That lets
us drop the union check #230302 added, which broke std::format.
Assisted-by: Cursor / Claude Opus 5.5
---
.../Transforms/CallConvLoweringPass.cpp | 70 -
.../TargetLowering/CIRABIRewriteContext.cpp | 636 +++++--
.../TargetLowering/CIRABIRewriteContext.h | 7 +-
clang/test/CIR/CodeGen/atomic.c | 8 +-
.../call-conv-lowering-x86_64-empty.cpp | 5 +-
...call-conv-lowering-x86_64-tail-padding.cpp | 90 +
.../call-conv-lowering-x86_64-union-copy.c | 641 +++++++
.../call-conv-lowering-x86_64-variadic.c | 76 +-
.../call-conv-lowering-x86_64-x87-union.cpp | 428 +++++
.../call-conv-lowering-x86_64-x87-vector.c | 5 +-
clang/test/CIR/CodeGen/call.c | 44 +-
clang/test/CIR/CodeGen/call.cpp | 12 +-
clang/test/CIR/CodeGen/cast.cpp | 6 +-
clang/test/CIR/CodeGen/complex.cpp | 32 +-
.../CodeGen/cxx-rewritten-binary-operator.cpp | 10 +-
.../test/CIR/CodeGen/global-decomp-decls.cpp | 11 +-
clang/test/CIR/CodeGen/instantiate-init.cpp | 20 +-
.../test/CIR/CodeGen/lambda-explicit-obj.cpp | 22 +-
clang/test/CIR/CodeGen/lambda.cpp | 6 +-
.../CIR/CodeGen/non-scalar-lval-return.cpp | 6 +-
.../CodeGen/pointer-to-member-func-cast.cpp | 36 +-
.../CodeGen/pointer-to-member-func-cmp.cpp | 24 +-
clang/test/CIR/CodeGen/struct.cpp | 25 +-
clang/test/CIR/CodeGen/ternary-throw.cpp | 571 +++----
clang/test/CIR/CodeGen/three-way-cmp.cpp | 10 +-
clang/test/CIR/CodeGen/union.c | 18 +-
clang/test/CIR/CodeGen/union.cpp | 8 +-
clang/test/CIR/CodeGen/var-arg-aggregate.c | 70 +-
.../CIR/CodeGen/var-arg-direct-offset.cpp | 15 +-
clang/test/CIR/CodeGenCoroutines/coro-agg.cpp | 6 +-
.../private-clause-coerced-ctor.cpp | 25 +-
.../abi-lowering/coerce-alloca-addrspace.cir | 1 -
.../abi-lowering/coerce-int-to-record.cir | 3 +-
.../coerce-record-return-from-memory.cir | 56 +-
.../coerce-record-return-larger.cir | 3 +-
.../abi-lowering/coerce-record-to-int.cir | 10 +-
.../coerce-record-to-record-via-memory.cir | 10 +-
.../abi-lowering/direct-flatten.cir | 71 +-
.../abi-lowering/indirect-byval.cir | 4 +-
.../indirect-non-byval-forward-param.cir | 8 +-
.../abi-lowering/indirect-return-sret.cir | 7 +-
.../abi-lowering/x86_64-aggregate-nyi.cir | 92 -
.../Transforms/abi-lowering/x86_64-bitint.cir | 126 +-
.../abi-lowering/x86_64-empty-class.cir | 4 +-
.../abi-lowering/x86_64-record-copy.cir | 1475 +++++++++++++++++
.../abi-lowering/x86_64-record-load-nyi.cir | 121 ++
.../x86_64-record-load-volatile.cir | 172 ++
.../abi-lowering/x86_64-record-packed.cir | 8 +-
.../x86_64-struct-direct-offset.cir | 13 +-
.../abi-lowering/x86_64-struct-direct.cir | 36 +-
.../abi-lowering/x86_64-struct-padded.cir | 14 +-
.../Transforms/abi-lowering/x86_64-union.cir | 445 ++++-
.../Transforms/abi-lowering/x86_64-vptr.cir | 5 +-
53 files changed, 4424 insertions(+), 1203 deletions(-)
create mode 100644 clang/test/CIR/CodeGen/call-conv-lowering-x86_64-tail-padding.cpp
create mode 100644 clang/test/CIR/CodeGen/call-conv-lowering-x86_64-union-copy.c
create mode 100644 clang/test/CIR/CodeGen/call-conv-lowering-x86_64-x87-union.cpp
create mode 100644 clang/test/CIR/Transforms/abi-lowering/x86_64-record-copy.cir
create mode 100644 clang/test/CIR/Transforms/abi-lowering/x86_64-record-load-nyi.cir
create mode 100644 clang/test/CIR/Transforms/abi-lowering/x86_64-record-load-volatile.cir
diff --git a/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp b/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp
index e7632bdcb700d2..aaacedeff1e74d 100644
--- a/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/CallConvLoweringPass.cpp
@@ -160,56 +160,6 @@ static bool hasIncompleteRecordByValue(mlir::Type ty) {
return false;
}
-/// Whether \p ty has the x87 double extended format, wrapped in LongDoubleType
-/// or not.
-static bool isX87DoubleExtended(mlir::Type ty) {
- auto fpTy = dyn_cast<cir::FPTypeInterface>(ty);
- return fpTy &&
- &fpTy.getFloatSemantics() == &llvm::APFloat::x87DoubleExtended();
-}
-
-/// Whether \p ty is, or holds by value, an x87 double extended value, alone or
-/// as the element of a _Complex or a vector.
-static bool holdsX87DoubleExtended(mlir::Type ty) {
- if (isa<cir::FPTypeInterface>(ty))
- return isX87DoubleExtended(ty);
- if (auto complexTy = dyn_cast<cir::ComplexType>(ty))
- return isX87DoubleExtended(complexTy.getElementType());
- if (auto vecTy = dyn_cast<cir::VectorType>(ty))
- return isX87DoubleExtended(vecTy.getElementType());
- if (auto arrTy = dyn_cast<cir::ArrayType>(ty))
- return arrTy.getSize() && holdsX87DoubleExtended(arrTy.getElementType());
- if (auto recTy = dyn_cast<cir::RecordType>(ty))
- return llvm::any_of(recTy.getMembers(), holdsX87DoubleExtended);
- return false;
-}
-
-/// Returns how many bytes from the start of \p ty a value of it reaches. A
-/// scalar reaches the end of its storage, a record the furthest end of its
-/// non-padding members, and an array the end of its last element, so padding
-/// at the end of a record does not count.
-static uint64_t getDataExtentInBytes(mlir::Type ty, const DataLayout &dl) {
- if (auto recTy = dyn_cast<cir::RecordType>(ty)) {
- uint64_t extent = 0;
- for (auto [idx, memberTy, kind] :
- llvm::enumerate(recTy.getMembers(), recTy.getMemberKinds())) {
- if (kind == cir::RecordMemberKind::Pad)
- continue;
- extent = std::max(extent, recTy.getElementOffset(dl, idx) +
- getDataExtentInBytes(memberTy, dl));
- }
- return extent;
- }
- if (auto arrTy = dyn_cast<cir::ArrayType>(ty)) {
- if (!arrTy.getSize())
- return 0;
- mlir::Type elemTy = arrTy.getElementType();
- return (arrTy.getSize() - 1) * dl.getTypeSize(elemTy).getFixedValue() +
- getDataExtentInBytes(elemTy, dl);
- }
- return dl.getTypeSize(ty).getFixedValue();
-}
-
/// The CIR types the x86_64 bridge handles. Scalars: an integer up to 128
/// bits (including `__int128`), a `_BitInt` of any width, pointer, vtable
/// pointer, bool, void, or any floating-point type. Aggregates: a complete
@@ -302,26 +252,6 @@ static bool isSupportedType(mlir::Type ty, const DataLayout &dl) {
// there is no member here to build the Indirect coercion from.
if (members.empty() && recordBits > 128)
return false;
- // A union is moved as a value of its storage type, and an x87 value
- // stores fewer bytes than it occupies, so the bytes another member
- // holds past them would be lost. A double extended member stores its
- // first 10 bytes, so it can share a union with other such members and
- // with members whose data ends within those bytes, provided it is the
- // storage type. An x87 value held any other way, in a _Complex, a
- // vector, a record or an array, is measured at its full storage, which
- // reaches past them.
- if (members.size() > 1 && llvm::any_of(members, holdsX87DoubleExtended)) {
- uint64_t x87StoreBytes = llvm::APFloat::semanticsSizeInBits(
- llvm::APFloat::x87DoubleExtended()) /
- 8;
- if (!isX87DoubleExtended(
- mlir::cast<cir::UnionType>(recTy).getUnionStorageType(dl)) ||
- !llvm::all_of(members, [&](mlir::Type m) {
- return isX87DoubleExtended(m) ||
- getDataExtentInBytes(m, dl) <= x87StoreBytes;
- }))
- return false;
- }
// A `_BitInt` access unit past an eightbyte has byte-array storage in
// CIR and integer storage in classic CodeGen, so the narrowing walk
// finds an i8 in the second eightbyte where classic keeps an i64.
diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
index aa8581056a80a8..7e167b66a32332 100644
--- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
@@ -51,12 +51,12 @@ using namespace mlir::abi;
// each member from the source alloca (get_member + load) when the operand is
// a load of an alloca, or via cir.extract_member otherwise.
//
-// For Direct + canFlatten (where the coerced type is a multi-field struct),
-// the coerced struct is similarly flattened into N individual wire arguments.
-// The callee reassembles the N scalar block args into the coerced struct,
-// then coerces to the original argument type if the two types differ. The
-// call site coerces the original type to the coerced struct, then extracts
-// each field as a separate call argument.
+// For Direct + canFlatten (where the coerced type is a multi-field struct), the
+// coerced struct is similarly passed as N separate arguments, one per field.
+// The callee stores the N scalar block args into one slot sized and aligned for
+// both the coerced struct and the original argument type, which the body reads.
+// The call site reads each field of the coerced struct out of memory holding
+// the operand as a separate call argument.
namespace {
@@ -103,10 +103,9 @@ buildNewArgTypes(ArrayRef<mlir::Type> oldArgTypes,
if (cir::RecordType flatTy = getFlattenedCoercedType(ac)) {
llvm::append_range(newArgTypes, flatTy.getMembers());
} else {
- // Direct with a coerced type: the wire signature uses the coerced
- // type; the body still expects origTy and insertArgCoercion recovers
- // it via a memory round-trip. Direct without coercion is a
- // pass-through.
+ // Direct with a coerced type: the lowered signature uses the coerced
+ // type; the body still expects origTy and insertArgCoercion recovers it
+ // from a coercion slot. Direct without coercion is a pass-through.
newArgTypes.push_back(ac.coercedType ? ac.coercedType : origTy);
}
break;
@@ -458,8 +457,8 @@ static cir::AllocaOp createCoercionSlot(
/// register tuple, must not be given an offset.
///
/// Any operations the helper creates are appended to \p createdOps so the
-/// caller can pass them to replaceAllUsesExcept and avoid clobbering the
-/// store's value operand when later rewiring the source value.
+/// caller can leave them out when it later rewires the source value's uses,
+/// which would otherwise clobber the store's value operand.
mlir::Value emitCoercionToMemory(mlir::OpBuilder &builder, mlir::Location loc,
mlir::Type dstTy, mlir::Value src,
mlir::Block *slotBlock,
@@ -540,6 +539,31 @@ static cir::LoadOp maybeGetSimpleLoad(mlir::Value val) {
return load;
}
+/// \p val's defining load, if it loads a record and is not atomic. Null
+/// otherwise. The load may be volatile, and then its bytes may only be read
+/// through a volatile copy, never in place.
+static cir::LoadOp maybeGetCopyableRecordLoad(mlir::Value val) {
+ cir::LoadOp load = val.getDefiningOp<cir::LoadOp>();
+ if (!load || load.getMemOrder() ||
+ !mlir::isa<cir::RecordType>(load.getType()))
+ return {};
+ return load;
+}
+
+/// Emits an error and returns failure if \p val, an operand of \p user, is an
+/// atomic load of a record. The record's bytes are read through a copy or in
+/// place, neither of which keeps the load atomic.
+static mlir::LogicalResult rejectAtomicRecordLoad(mlir::Operation *user,
+ mlir::Value val) {
+ auto load = val.getDefiningOp<cir::LoadOp>();
+ if (!load || !load.getMemOrder() ||
+ !mlir::isa<cir::RecordType>(load.getType()))
+ return mlir::success();
+ return user->emitOpError()
+ << "atomic load of a record operand not yet implemented in "
+ "CallConvLowering";
+}
+
/// Whether the storage the record load \p load reads is an alloca that holds
/// \p coercedTy at \p offset. With no offset, the coerced type's alloc size
/// has to fit in the record's. At an offset, the coerced value has to end
@@ -574,17 +598,16 @@ static mlir::Value emitInPlaceCoercedLoad(mlir::OpBuilder &builder,
return coerced;
}
-/// Copy the bytes of the record \p load reads into a new coercion slot at the
-/// start of \p slotBlock, then read \p coercedTy out of the slot at \p offset.
-/// The slot is at least as aligned as the storage copied from, and the read
-/// takes the slot's alignment at \p offset. The copy goes just before
-/// \p load, so it reads the bytes \p load reads.
-static mlir::Value emitCopiedCoercedLoad(mlir::OpBuilder &builder,
- mlir::Location loc, cir::LoadOp load,
- mlir::Type coercedTy,
- mlir::Block *slotBlock,
- const mlir::DataLayout &dl,
- unsigned offset) {
+/// Copy the bytes of the record \p load reads into a new coercion slot for
+/// \p coercedTy at \p offset, placed at the start of \p slotBlock, and return
+/// the slot. The slot is at least as aligned as the storage copied from. The
+/// copy goes just before \p load, so it reads the bytes \p load reads, and is
+/// volatile if \p load is.
+static cir::AllocaOp
+emitCopiedCoercionSlot(mlir::OpBuilder &builder, mlir::Location loc,
+ cir::LoadOp load, mlir::Type coercedTy,
+ mlir::Block *slotBlock, const mlir::DataLayout &dl,
+ unsigned offset) {
auto recTy = cast<cir::RecordType>(load.getType());
uint64_t srcAlign;
if (cir::AllocaOp src = cir::getUnderlyingAlloca(load.getAddr()))
@@ -599,10 +622,30 @@ static mlir::Value emitCopiedCoercedLoad(mlir::OpBuilder &builder,
"a direct offset must leave the record as the slot's type");
mlir::OpBuilder::InsertionGuard guard(builder);
builder.setInsertionPoint(load);
- mlir::Value recView = emitViewAtOffset(builder, loc, recTy, slot, ignored, 0);
- cir::CopyOp::create(builder, loc, recView, load.getAddr(),
- builder.getI64IntegerAttr(slot.getAlignment()),
- builder.getI64IntegerAttr(srcAlign));
+ mlir::Value recView =
+ emitViewAtOffset(builder, loc, recTy, slot, ignored, /*offset=*/0);
+ auto copy =
+ cir::CopyOp::create(builder, loc, recView, load.getAddr(),
+ builder.getI64IntegerAttr(slot.getAlignment()),
+ builder.getI64IntegerAttr(srcAlign));
+ copy.setIsVolatile(load.getIsVolatile());
+ return slot;
+}
+
+/// Copy the bytes of the record \p load reads into a coercion slot, as
+/// emitCopiedCoercionSlot does, then read \p coercedTy out of the slot at
+/// \p offset. The read takes the slot's alignment at \p offset.
+static mlir::Value emitCopiedCoercedLoad(mlir::OpBuilder &builder,
+ mlir::Location loc, cir::LoadOp load,
+ mlir::Type coercedTy,
+ mlir::Block *slotBlock,
+ const mlir::DataLayout &dl,
+ unsigned offset) {
+ cir::AllocaOp slot = emitCopiedCoercionSlot(builder, loc, load, coercedTy,
+ slotBlock, dl, offset);
+ mlir::OpBuilder::InsertionGuard guard(builder);
+ builder.setInsertionPoint(load);
+ SmallPtrSet<mlir::Operation *, 4> ignored;
mlir::Value coercedView =
emitViewAtOffset(builder, loc, coercedTy, slot, ignored, offset);
auto coerced = cir::LoadOp::create(builder, loc, coercedView);
@@ -610,18 +653,178 @@ static mlir::Value emitCopiedCoercedLoad(mlir::OpBuilder &builder,
return coerced;
}
+/// Read \p coercedTy at \p offset out of the bytes the record load \p load
+/// reads: straight out of its storage if \p load is not volatile and
+/// canReadInPlace accepts it, and otherwise out of a coercion slot at the start
+/// of \p slotBlock that those bytes are copied into. emitInPlaceCoercedLoad
+/// does not mark its read volatile, so a volatile load is read through the
+/// volatile copy instead. Either way no new load of the record type is
+/// emitted.
+static mlir::Value emitCoercedRecordLoad(
+ mlir::OpBuilder &builder, mlir::Location loc, cir::LoadOp load,
+ mlir::Type coercedTy, mlir::Block *slotBlock, const mlir::DataLayout &dl,
+ const cir::CIRDataLayout &cirDataLayout, unsigned offset) {
+ if (!load.getIsVolatile() &&
+ canReadInPlace(load, coercedTy, cirDataLayout, offset))
+ return emitInPlaceCoercedLoad(builder, loc, load, coercedTy, offset);
+ return emitCopiedCoercedLoad(builder, loc, load, coercedTy, slotBlock, dl,
+ offset);
+}
+
+/// Whether \p ty is a struct that ends in pad members, as CIRGen emits for tail
+/// padding another object may reuse. It is also true of other structs CIRGen
+/// ends in pad members, such as an over-aligned one.
+static bool hasTailPadding(mlir::Type ty) {
+ auto structTy = mlir::dyn_cast<cir::StructType>(ty);
+ if (!structTy)
+ return false;
+ llvm::ArrayRef<mlir::Type> members = structTy.getMembers();
+ llvm::ArrayRef<cir::RecordMemberKind> kinds = structTy.getMemberKinds();
+ for (auto [member, kind] : llvm::reverse(llvm::zip_equal(members, kinds)))
+ if (cir::memberOwnsBytes(member))
+ return kind == cir::RecordMemberKind::Pad;
+ return false;
+}
+
+/// Whether a cir.copy of \p ty into \p dst that stands in for a store of it
+/// skips tail padding. Unless \p dst is an alloca, which holds a complete
+/// object, it can be a potentially-overlapping subobject, such as a
+/// [[no_unique_address]] member or a base, whose tail padding holds another
+/// object. Classic CodeGen writes only the data size into a destination it is
+/// handed (EmitCall) and into one that may overlap (EmitAggregateCopy).
+/// Copying the whole object into an alloca is equivalent, since nothing else
+/// lives in its tail padding.
+static bool shouldSkipTailPadding(mlir::Value dst, mlir::Type ty) {
+ return !cir::getUnderlyingAlloca(dst) && hasTailPadding(ty);
+}
+
+/// Rewire the uses of the record value \p val, whose bytes sit at \p addr,
+/// which is aligned to \p addrAlign, so that each one finds the record in
+/// memory. \p val may already have been retyped to the type it is passed as, a
+/// coerced type or a pointer, but its uses still expect the record.
+///
+/// A store of \p val becomes a cir.copy from \p addr at the store, volatile if
+/// the store is and skipping tail padding as shouldSkipTailPadding says. A
+/// store of the record value writes only what its lowered type stores, so
+/// another union member's data past those bytes would be lost. A consumer may
+/// be rewritten before \p val's defining op, and has then already stored \p val
+/// into a slot of its own. That store becomes a copy here too.
+///
+/// Every other use is pointed at one cir.load of \p addr created at the
+/// builder's insertion point. Only a call argument, a return and a read of a
+/// member that is not itself a record may use it, since those either read
+/// \p addr when they are rewritten or read a field. Uses by an operation in
+/// \p exceptions are left alone. Fails, after reporting it, on an atomic store
+/// of \p val or a use by anything else, before anything is rewired.
+static mlir::LogicalResult
+rewireRecordUses(mlir::OpBuilder &builder, mlir::Location loc, mlir::Value val,
+ mlir::Value addr, uint64_t addrAlign,
+ const mlir::DataLayout &dl,
+ const SmallPtrSetImpl<mlir::Operation *> &exceptions) {
+ mlir::Type recTy = mlir::cast<cir::PointerType>(addr.getType()).getPointee();
+ assert(mlir::isa<cir::RecordType>(recTy) && "only a record is rewired");
+ SmallVector<cir::StoreOp> stores;
+ SmallVector<mlir::OpOperand *> otherUses;
+ for (mlir::OpOperand &use : val.getUses()) {
+ mlir::Operation *user = use.getOwner();
+ if (exceptions.contains(user))
+ continue;
+ auto store = mlir::dyn_cast<cir::StoreOp>(user);
+ if (!store) {
+ auto member = mlir::dyn_cast<cir::ExtractMemberOp>(user);
+ bool isFieldRead =
+ member && !mlir::isa<cir::RecordType>(member.getType());
+ if (!isFieldRead &&
+ !mlir::isa<cir::CIRCallOpInterface, cir::ReturnOp>(user))
+ return user->emitOpError()
+ << "use of a record argument or result other than by a "
+ "store, call, return or member read not yet implemented "
+ "in CallConvLowering";
+ otherUses.push_back(&use);
+ continue;
+ }
+ if (store.getMemOrder())
+ return store.emitOpError()
+ << "atomic store of a record argument or result not yet "
+ "implemented in CallConvLowering";
+ stores.push_back(store);
+ }
+
+ mlir::IntegerAttr srcAlign = builder.getI64IntegerAttr(addrAlign);
+ for (cir::StoreOp store : stores) {
+ mlir::OpBuilder::InsertionGuard guard(builder);
+ builder.setInsertionPoint(store);
+ uint64_t dstAlign =
+ store.getAlignment().value_or(dl.getTypeABIAlignment(recTy));
+ auto copy =
+ cir::CopyOp::create(builder, store.getLoc(), store.getAddr(), addr,
+ builder.getI64IntegerAttr(dstAlign), srcAlign);
+ copy.setIsVolatile(store.getIsVolatile());
+ copy.setSkipTailPadding(shouldSkipTailPadding(store.getAddr(), recTy));
+ store->erase();
+ }
+
+ if (otherUses.empty())
+ return mlir::success();
+ auto load = cir::LoadOp::create(builder, loc, addr);
+ load.setAlignment(addrAlign);
+ for (mlir::OpOperand *use : otherUses)
+ use->set(load);
+ return mlir::success();
+}
+
+/// Point the uses of \p val outside \p exceptions at the value of \p val's
+/// original type that \p view, a view of the start of a coercion slot, holds.
+/// For a record, rewireRecordUses points the uses at the slot. Any other type
+/// is loaded out of it at the builder's insertion point.
+static mlir::LogicalResult
+replaceUsesFromSlot(mlir::OpBuilder &builder, mlir::Location loc,
+ mlir::Value val, mlir::Value view,
+ const mlir::DataLayout &dl,
+ const SmallPtrSetImpl<mlir::Operation *> &exceptions) {
+ if (!mlir::isa<cir::RecordType>(
+ mlir::cast<cir::PointerType>(view.getType()).getPointee())) {
+ auto load = cir::LoadOp::create(builder, loc, view);
+ val.replaceAllUsesExcept(load, exceptions);
+ return mlir::success();
+ }
+ cir::AllocaOp slot = cir::getUnderlyingAlloca(view);
+ assert(slot && "the view must start at the coercion slot");
+ return rewireRecordUses(builder, loc, val, view, slot.getAlignment(), dl,
+ exceptions);
+}
+
+/// Coerce \p src, which is \p val retyped or the value replacing it, back to
+/// \p origTy through a coercion slot at the start of \p slotBlock, and point
+/// the uses of \p val outside \p createdOps at the result with
+/// replaceUsesFromSlot. A direct \p offset lands on the coerced side, so the
+/// view of \p origTy starts at the slot. The coercion's operations are added
+/// to \p createdOps.
+static mlir::LogicalResult
+emitCoercionAndReplaceUses(mlir::OpBuilder &builder, mlir::Location loc,
+ mlir::Type origTy, mlir::Value src, mlir::Value val,
+ mlir::Block *slotBlock, const mlir::DataLayout &dl,
+ SmallPtrSetImpl<mlir::Operation *> &createdOps,
+ unsigned offset) {
+ mlir::Value view = emitCoercionToMemory(builder, loc, origTy, src, slotBlock,
+ dl, createdOps, offset);
+ return replaceUsesFromSlot(builder, loc, val, view, dl, createdOps);
+}
+
/// Coerce the value each cir.return returns to the new (coerced) return type.
///
-/// When the returned value is a simple load of a record, the coerced value is
-/// read straight out of the storage the load reads if canReadInPlace accepts
-/// it, and otherwise out of a coercion slot the record's bytes are copied into.
-/// The record load is erased once nothing else uses it. Any other value is
-/// stored into a coercion slot and read back.
-void insertReturnCoercion(mlir::FunctionOpInterface funcOp,
- mlir::Type origRetTy, mlir::Type coercedRetTy,
- mlir::OpBuilder &builder, const mlir::DataLayout &dl,
- unsigned offset) {
- cir::CIRDataLayout cirDataLayout(funcOp->getParentOfType<mlir::ModuleOp>());
+/// When the returned value is a non-atomic load of a record, the coerced value
+/// is read by emitCoercedRecordLoad, and the record load is erased once nothing
+/// else uses it. Any other value is stored into a coercion slot and read back.
+/// A record with no storage behind it, such as a constant, defines no bytes
+/// beyond its fields, so the store writes every byte it defines. A call
+/// result's store is turned into a copy when its call is rewritten later.
+/// Fails, after reporting it, on an atomic load of a record.
+mlir::LogicalResult
+insertReturnCoercion(mlir::FunctionOpInterface funcOp, mlir::Type origRetTy,
+ mlir::Type coercedRetTy, mlir::OpBuilder &builder,
+ const mlir::DataLayout &dl,
+ const cir::CIRDataLayout &cirDataLayout, unsigned offset) {
SmallVector<cir::ReturnOp> returns;
funcOp.walk([&](cir::ReturnOp r) { returns.push_back(r); });
for (cir::ReturnOp r : returns) {
@@ -631,15 +834,12 @@ void insertReturnCoercion(mlir::FunctionOpInterface funcOp,
if (origVal.getType() == coercedRetTy)
continue;
- cir::LoadOp recLoad = maybeGetSimpleLoad(origVal);
- if (recLoad && mlir::isa<cir::RecordType>(recLoad.getType())) {
- mlir::Value coerced =
- canReadInPlace(recLoad, coercedRetTy, cirDataLayout, offset)
- ? emitInPlaceCoercedLoad(builder, r.getLoc(), recLoad,
- coercedRetTy, offset)
- : emitCopiedCoercedLoad(
- builder, r.getLoc(), recLoad, coercedRetTy,
- &funcOp->getRegion(0).front(), dl, offset);
+ if (failed(rejectAtomicRecordLoad(r, origVal)))
+ return mlir::failure();
+ if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(origVal)) {
+ mlir::Value coerced = emitCoercedRecordLoad(
+ builder, r.getLoc(), recLoad, coercedRetTy,
+ &funcOp->getRegion(0).front(), dl, cirDataLayout, offset);
r->setOperand(0, coerced);
if (recLoad->use_empty())
recLoad->erase();
@@ -652,6 +852,7 @@ void insertReturnCoercion(mlir::FunctionOpInterface funcOp,
&funcOp->getRegion(0).front(), dl, offset);
r->setOperand(0, coerced);
}
+ return mlir::success();
}
/// \p recordVal's defining load, if it is simple and its address resolves to
@@ -804,32 +1005,35 @@ findParamSpill(mlir::BlockArgument blockArg) {
return {store, cast<cir::AllocaOp>(store.getAddr().getDefiningOp())};
}
-/// For each Direct arg with a coerced type, change the block argument's type
-/// to the coerced type and insert a coercion at function entry that maps it
-/// back to the original type for body uses. For each Indirect byval arg,
-/// change the block argument's type to a pointer, then replace a record's
-/// param spill with a copy from that pointer, or insert a load at entry for
-/// a body that reads the parameter as a value. For each Indirect non-byval
-/// arg, change the block argument to a pointer and queue the param-slot
-/// alloca to be replaced by it (no entry load or byte-copy) so the body
-/// operates on the caller's storage in place. For each Expand arg, replace
-/// the single struct block argument with N scalar block arguments (one per
-/// field) and store each field directly into the parameter's own alloca (the
-/// CIRGen spill slot), erasing the original whole-struct store.
+/// For each Direct arg with a coerced type, change the block argument's type to
+/// the coerced type and insert a coercion at function entry that maps it back
+/// to the original type for body uses, which find a record in the coercion slot
+/// through rewireRecordUses. For each Indirect byval arg, change the block
+/// argument's type to a pointer, then replace a record's single param spill
+/// with a copy from that pointer. Without such a spill, a record's uses find
+/// it at the pointer through rewireRecordUses, and a value of any other type is
+/// loaded from the pointer at entry. For each Indirect non-byval arg, change
+/// the block argument to a pointer and queue the param-slot alloca to be
+/// replaced by it (no entry load or byte-copy) so the body operates on the
+/// caller's storage in place. For each Expand arg, replace the single struct
+/// block argument with N scalar block arguments (one per field) and store each
+/// field directly into the parameter's own alloca (the CIRGen spill slot),
+/// erasing the original whole-struct store.
///
/// \p hasSRetArg is true when the function has an sret return (a hidden return
/// pointer is prepended as block argument 0). Expand arguments expand the
/// block argument count, so a running index tracks the current block argument
/// position rather than computing the classification index + \p hasSRetArg
-/// directly.
-void insertArgCoercion(
- mlir::FunctionOpInterface funcOp, const FunctionClassification &fc,
- mlir::OpBuilder &builder, const mlir::DataLayout &dl, bool hasSRetArg,
- SmallVectorImpl<std::pair<cir::AllocaOp, mlir::BlockArgument>>
- &pendingParamSlots) {
+/// directly. Fails, after reporting it, when rewireRecordUses does.
+mlir::LogicalResult
+insertArgCoercion(mlir::FunctionOpInterface funcOp,
+ const FunctionClassification &fc, mlir::OpBuilder &builder,
+ const mlir::DataLayout &dl, bool hasSRetArg,
+ SmallVectorImpl<std::pair<cir::AllocaOp, mlir::BlockArgument>>
+ &pendingParamSlots) {
mlir::Region &body = funcOp->getRegion(0);
if (body.empty())
- return;
+ return mlir::success();
mlir::Block &entry = body.front();
// Running block argument index. Each non-Expand classification occupies
@@ -919,12 +1123,11 @@ void insertArgCoercion(
mlir::BlockArgument blockArg = entry.getArgument(blockArgIdx);
if (cir::RecordType flatTy = getFlattenedCoercedType(ac)) {
- // Direct + canFlatten: the coerced type is a struct whose fields become
- // individual wire arguments. The reconstruction mirrors the Expand path
- // — replace the single block arg with N scalar block args, store them
- // into an alloca of the coerced struct type, reload — but then applies
- // an additional coercion from the coerced struct type to the original
- // argument type if the two differ in layout.
+ // Direct + canFlatten: the coerced type is a struct whose fields are
+ // passed as separate arguments. As in the Expand path, the single block
+ // arg becomes N scalar block args, and each is stored straight into a
+ // slot sized and aligned for both the coerced struct and the original
+ // argument type, which the body then reads.
unsigned numFields = flatTy.getNumElements();
assert(numFields >= 2 && "getFlattenedCoercedType guarantees >1 fields");
Type origTy = blockArg.getType();
@@ -937,43 +1140,43 @@ void insertArgCoercion(
// setInsertionPointToStart: see comment in the Expand arm above.
builder.setInsertionPointToStart(&entry);
- cir::PointerType flatPtrTy = getAllocaPtrTy(flatTy, dl);
- uint64_t flatAlign = dl.getTypeABIAlignment(flatTy);
- auto flatSlot = cir::AllocaOp::create(
- builder, loc, flatPtrTy, builder.getStringAttr("coerce"),
- builder.getI64IntegerAttr(flatAlign));
- SmallPtrSet<Operation *, 8> flattenOps = {flatSlot};
+ assert(!ac.directOffset &&
+ "the coerced struct starts at slot offset 0 here, so a flattened "
+ "coercion cannot honor a direct offset");
+ SmallPtrSet<Operation *, 8> flattenOps;
+ cir::AllocaOp slot;
+ if (origTy == flatTy) {
+ slot = cir::AllocaOp::create(
+ builder, loc, getAllocaPtrTy(flatTy, dl),
+ builder.getStringAttr("coerce"),
+ builder.getI64IntegerAttr(dl.getTypeABIAlignment(flatTy)));
+ flattenOps.insert(slot);
+ } else {
+ slot = createCoercionSlot(builder, loc, flatTy, origTy, &entry, dl,
+ flattenOps, /*offset=*/0);
+ }
+ Value flatView = emitViewAtOffset(builder, loc, flatTy, slot, flattenOps,
+ /*offset=*/0);
+ auto flatPtrTy = cast<cir::PointerType>(flatView.getType());
for (auto [f, fieldTy] : llvm::enumerate(flatTy.getMembers())) {
Type fieldPtrTy =
cir::PointerType::get(fieldTy, flatPtrTy.getAddrSpace());
auto fieldPtr = cir::GetMemberOp::create(builder, loc, fieldPtrTy,
- flatSlot, /*name=*/"",
+ flatView, /*name=*/"",
/*index=*/f);
flattenOps.insert(fieldPtr);
auto storeOp = cir::StoreOp::create(
builder, loc, entry.getArgument(blockArgIdx + f), fieldPtr);
flattenOps.insert(storeOp);
}
- auto flatLoaded =
- cir::LoadOp::create(builder, loc, flatTy, flatSlot.getResult());
- flattenOps.insert(flatLoaded);
-
- // If the coerced struct type differs from the original argument type,
- // insert a memory round-trip to recover the original type for body uses.
- Value finalVal = flatLoaded;
- if (origTy != flatTy) {
- SmallPtrSet<Operation *, 4> coercionOps;
- assert(!ac.directOffset &&
- "each field is read from slot offset 0 here, so a flattened "
- "coercion cannot honor a direct offset");
- finalVal = emitCoercion(builder, loc, origTy, flatLoaded, &entry, dl,
- coercionOps, /*offset=*/0);
- flattenOps.insert(coercionOps.begin(), coercionOps.end());
- }
- // Replace all original body uses of the struct block arg (now field 0)
- // with the recovered original-type value.
- blockArg.replaceAllUsesExcept(finalVal, flattenOps);
+ // Point the original body uses of the struct block arg, now field 0, at
+ // the original-type view of the slot.
+ Value origView = emitViewAtOffset(builder, loc, origTy, slot, flattenOps,
+ /*offset=*/0);
+ if (failed(replaceUsesFromSlot(builder, loc, blockArg, origView, dl,
+ flattenOps)))
+ return mlir::failure();
blockArgIdx += numFields;
continue;
@@ -988,17 +1191,14 @@ void insertArgCoercion(
}
blockArg.setType(newArgTy);
+ // The coercion's own operations are left out of the rewiring, since they
+ // read blockArg as the coerced type.
builder.setInsertionPointToStart(&entry);
SmallPtrSet<mlir::Operation *, 4> coercionOps;
- mlir::Value adapted =
- emitCoercion(builder, funcOp.getLoc(), oldArgTy, blockArg, &entry, dl,
- coercionOps, ac.directOffset);
-
- // Replace blockArg uses with the adapted value, except inside the
- // helper ops we just created. This is critical: the StoreOp's value
- // operand is blockArg, and if we naively replaceAllUses it gets swapped
- // to adapted (now of the original type != the alloca's pointee type).
- blockArg.replaceAllUsesExcept(adapted, coercionOps);
+ if (failed(emitCoercionAndReplaceUses(builder, funcOp.getLoc(), oldArgTy,
+ blockArg, blockArg, &entry, dl,
+ coercionOps, ac.directOffset)))
+ return mlir::failure();
} else if (ac.kind == ArgKind::Indirect) {
// byval and non-byval both lower to !cir.ptr<T>, and which it is shows
// up only in the attrs updateArgAttrs applies. Body lowering differs:
@@ -1045,9 +1245,17 @@ void insertArgCoercion(
mlir::Value destAddr;
mlir::IntegerAttr destAlign;
mlir::Operation *spillPos = nullptr;
+ bool spillIsVolatile = false;
if (paramStore) {
+ if (paramStore.getMemOrder())
+ return paramStore.emitOpError()
+ << "atomic store of a record argument or result not yet "
+ "implemented in CallConvLowering";
+ spillIsVolatile = paramStore.getIsVolatile();
destAddr = paramStore.getAddr();
- if (cir::AllocaOp destAlloca = cir::getUnderlyingAlloca(destAddr))
+ cir::AllocaOp destAlloca = cir::getUnderlyingAlloca(destAddr);
+ destAlign = paramStore.getAlignmentAttr();
+ if (!destAlign && destAlloca)
destAlign = destAlloca.getAlignmentAttr();
spillPos = paramStore->getNextNode();
assert(spillPos && "param spill must be followed by a block "
@@ -1060,15 +1268,26 @@ void insertArgCoercion(
if (destAddr) {
mlir::OpBuilder::InsertionGuard guard(builder);
builder.setInsertionPoint(spillPos);
- cir::CopyOp::create(
+ auto copy = cir::CopyOp::create(
builder, funcOp.getLoc(), destAddr, blockArg, destAlign,
builder.getI64IntegerAttr(ac.indirectAlign.value()));
+ copy.setIsVolatile(spillIsVolatile);
+ copy.setSkipTailPadding(
+ shouldSkipTailPadding(destAddr, ptrTy.getPointee()));
} else if (!blockArg.use_empty()) {
- // No spill to replace, so materialize the value the body reads.
builder.setInsertionPointToStart(&entry);
- auto loadOp = cir::LoadOp::create(builder, funcOp.getLoc(), blockArg);
- SmallPtrSet<mlir::Operation *, 1> loadOps = {loadOp};
- blockArg.replaceAllUsesExcept(loadOp.getResult(), loadOps);
+ if (mlir::isa<cir::RecordType>(ptrTy.getPointee())) {
+ SmallPtrSet<mlir::Operation *, 1> noExceptions;
+ if (failed(rewireRecordUses(builder, funcOp.getLoc(), blockArg,
+ blockArg, ac.indirectAlign.value(), dl,
+ noExceptions)))
+ return mlir::failure();
+ } else {
+ auto loadOp =
+ cir::LoadOp::create(builder, funcOp.getLoc(), blockArg);
+ SmallPtrSet<mlir::Operation *, 1> loadOps = {loadOp};
+ blockArg.replaceAllUsesExcept(loadOp.getResult(), loadOps);
+ }
}
}
}
@@ -1076,6 +1295,7 @@ void insertArgCoercion(
++blockArgIdx;
}
+ return mlir::success();
}
/// Rewrite each cir.return so the return value flows through the sret
@@ -1252,21 +1472,21 @@ static void prependIndirectCallee(cir::CallOp call,
args.insert(args.begin(), calleePtr);
}
-/// Rewrite an indirect-return (sret) call site: prepend a return-slot
-/// pointer as operand 0, make the call return void, and either reuse a
-/// dominating single-use store destination as the slot (so construction
-/// flows directly into it) or allocate a fresh slot and load the result
-/// back out. \p newArgs is the already-shaped (Ignore-dropped,
+/// Rewrite an indirect-return (sret) call site: prepend a return-slot pointer
+/// as operand 0, make the call return void, and either reuse a dominating
+/// single-use store destination as the slot (so construction flows directly
+/// into it) or allocate a fresh slot and point the result's uses at it, a
+/// record's through rewireRecordUses and any other type's through a load of the
+/// slot after the call. \p newArgs is the already-shaped (Ignore-dropped,
/// coercion-applied) non-sret argument list, the last \p numEllipsisArgs of
-/// which were passed through the callee's ellipsis. The caller guarantees
-/// the call has a result and an indirect-return classification.
-void rewriteIndirectReturnCall(cir::CallOp call,
- const FunctionClassification &fc,
- ArrayRef<mlir::Value> newArgs,
- unsigned numEllipsisArgs, mlir::Type origRetTy,
- ArrayRef<mlir::Type> origCallArgTypes,
- mlir::OpBuilder &builder,
- const mlir::DataLayout &dl) {
+/// which were passed through the callee's ellipsis. The caller guarantees the
+/// call has a result and an indirect-return classification. Fails, after
+/// reporting it, when rewireRecordUses does.
+mlir::LogicalResult rewriteIndirectReturnCall(
+ cir::CallOp call, const FunctionClassification &fc,
+ ArrayRef<mlir::Value> newArgs, unsigned numEllipsisArgs,
+ mlir::Type origRetTy, ArrayRef<mlir::Type> origCallArgTypes,
+ mlir::OpBuilder &builder, const mlir::DataLayout &dl) {
mlir::MLIRContext *ctx = call->getContext();
auto ptrTy = cir::PointerType::get(origRetTy);
builder.setInsertionPoint(call);
@@ -1329,6 +1549,12 @@ void rewriteIndirectReturnCall(cir::CallOp call,
// original store-from-result is redundant; dropping it avoids a
// byte-copy of the record.
reuseStore->erase();
+ } else if (mlir::isa<cir::RecordType>(origRetTy)) {
+ builder.setInsertionPointAfter(newCall);
+ SmallPtrSet<mlir::Operation *, 1> noExceptions;
+ if (failed(rewireRecordUses(builder, call.getLoc(), call.getResult(),
+ sretSlot, sretAlign, dl, noExceptions)))
+ return mlir::failure();
} else {
builder.setInsertionPointAfter(newCall);
auto load = cir::LoadOp::create(builder, call.getLoc(), origRetTy, sretSlot,
@@ -1342,6 +1568,7 @@ void rewriteIndirectReturnCall(cir::CallOp call,
call.getResult().replaceAllUsesWith(load);
}
call->erase();
+ return mlir::success();
}
/// Whether \p ty, a type the classifier named for one register of a
@@ -1590,15 +1817,19 @@ mlir::LogicalResult CIRABIRewriteContext::rewriteFunctionDefinition(
// becomes a pointer, and an Expand arg becomes one block argument per
// field. Done before the Ignore-drop below so the entry block
// argument indices used here still refer to the original positions.
- insertArgCoercion(funcOp, fc, builder, dl, hasSRet, pendingParamSlots);
+ if (failed(insertArgCoercion(funcOp, fc, builder, dl, hasSRet,
+ pendingParamSlots)))
+ return mlir::failure();
// Direct return with coerced type: insert a coercion at every
// cir.return so the returned value matches the (coerced) return
// type in the new function signature set below.
if (fc.returnInfo.kind == ArgKind::Direct && fc.returnInfo.coercedType &&
- !oldResultTypes.empty() && fc.returnInfo.coercedType != origRetTy)
- insertReturnCoercion(funcOp, origRetTy, fc.returnInfo.coercedType,
- builder, dl, fc.returnInfo.directOffset);
+ !oldResultTypes.empty() && fc.returnInfo.coercedType != origRetTy &&
+ failed(insertReturnCoercion(
+ funcOp, origRetTy, fc.returnInfo.coercedType, builder, dl,
+ cirDataLayout, fc.returnInfo.directOffset)))
+ return mlir::failure();
mlir::Block &entry = body.front();
@@ -1734,10 +1965,10 @@ CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp,
mlir::ValueRange argOperands = call.getArgOperands();
newArgs.reserve(argOperands.size());
- // Loads that the new call leaves unused: Expand and Direct+canFlatten read
- // the fields out of the source alloca, and an Indirect argument passes or
- // copies from the address the load read from. The old call still uses
- // them, so erase them only after it is gone.
+ // Loads that the new call leaves unused: a Direct (coerced or flattened) or
+ // Expand record is read out of its load's storage or a copy of it, and an
+ // Indirect argument passes or copies from the address the load read from.
+ // The old call still uses them, so erase them only after it is gone.
SmallVector<cir::LoadOp> deadRecordLoads;
// Capture original arg types before building newArgs (an Indirect argument
@@ -1767,19 +1998,41 @@ CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp,
mlir::Value arg = argOperands[idx];
if (cir::RecordType flatTy = getFlattenedCoercedType(ac)) {
// Direct + canFlatten: pass one scalar call argument per field of the
- // ABI-coerced struct. When the original and coerced types differ in
- // layout, coerce through a memory slot and read each field with
- // cir.get_member + cir.load from that slot. When the types already
- // match, decompose the struct value directly (reading from its source
- // alloca when possible).
+ // ABI-coerced struct. A non-atomic record load is read in place or
+ // through a copy of its storage, as emitCoercedRecordLoad decides, and
+ // any other operand is stored into a coercion slot first. When the types
+ // already match, the fields come from the struct value directly (reading
+ // from its source alloca when possible).
if (arg.getType() != flatTy) {
SmallPtrSet<mlir::Operation *, 4> coercionOps;
assert(!ac.directOffset &&
- "each field is read from slot offset 0 here, so a flattened "
- "coercion cannot honor a direct offset");
- mlir::Value coercedPtr =
- emitCoercionToMemory(builder, call.getLoc(), flatTy, arg, slotBlock,
- dl, coercionOps, /*offset=*/0);
+ "the coerced struct starts at slot offset 0 here, so a "
+ "flattened coercion cannot honor a direct offset");
+ if (failed(rejectAtomicRecordLoad(call, arg)))
+ return mlir::failure();
+ mlir::OpBuilder::InsertionGuard guard(builder);
+ mlir::Value coercedPtr;
+ uint64_t baseAlign = 0;
+ if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(arg)) {
+ mlir::Value base = recLoad.getAddr();
+ if (!recLoad.getIsVolatile() &&
+ canReadInPlace(recLoad, flatTy, cirDataLayout, /*offset=*/0)) {
+ cir::AllocaOp baseAlloca = cir::getUnderlyingAlloca(base);
+ assert(baseAlloca && "canReadInPlace requires an alloca");
+ baseAlign = baseAlloca.getAlignment();
+ } else {
+ base = emitCopiedCoercionSlot(builder, call.getLoc(), recLoad,
+ flatTy, slotBlock, dl, /*offset=*/0);
+ }
+ builder.setInsertionPoint(recLoad);
+ coercedPtr = emitViewAtOffset(builder, call.getLoc(), flatTy, base,
+ coercionOps, /*offset=*/0);
+ deadRecordLoads.push_back(recLoad);
+ } else {
+ coercedPtr =
+ emitCoercionToMemory(builder, call.getLoc(), flatTy, arg,
+ slotBlock, dl, coercionOps, /*offset=*/0);
+ }
auto coercedPtrTy = mlir::cast<cir::PointerType>(coercedPtr.getType());
for (auto [f, fieldTy] : llvm::enumerate(flatTy.getMembers())) {
mlir::Type fieldPtrTy =
@@ -1787,8 +2040,14 @@ CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp,
auto fieldPtr =
cir::GetMemberOp::create(builder, call.getLoc(), fieldPtrTy,
coercedPtr, /*name=*/"", /*index=*/f);
- newArgs.push_back(cir::LoadOp::create(builder, call.getLoc(), fieldTy,
- fieldPtr.getResult()));
+ auto field = cir::LoadOp::create(builder, call.getLoc(), fieldTy,
+ fieldPtr.getResult());
+ // Storage read in place may be less aligned than the coerced struct,
+ // so each field claims only what its offset in it allows.
+ if (baseAlign)
+ field.setAlignment(
+ llvm::MinAlign(baseAlign, flatTy.getElementOffset(dl, f)));
+ newArgs.push_back(field);
}
} else {
emitStructFieldArgs(builder, call.getLoc(), arg, flatTy, newArgs,
@@ -1804,9 +2063,17 @@ CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp,
deadRecordLoads);
} else if (ac.kind == ArgKind::Direct && ac.coercedType &&
arg.getType() != ac.coercedType) {
- arg = emitCoercion(builder, call.getLoc(), ac.coercedType, arg, slotBlock,
- dl, ac.directOffset);
- newArgs.push_back(arg);
+ if (failed(rejectAtomicRecordLoad(call, arg)))
+ return mlir::failure();
+ if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(arg)) {
+ newArgs.push_back(emitCoercedRecordLoad(
+ builder, call.getLoc(), recLoad, ac.coercedType, slotBlock, dl,
+ cirDataLayout, ac.directOffset));
+ deadRecordLoads.push_back(recLoad);
+ } else {
+ newArgs.push_back(emitCoercion(builder, call.getLoc(), ac.coercedType,
+ arg, slotBlock, dl, ac.directOffset));
+ }
} else if (ac.kind == ArgKind::Indirect) {
uint64_t minAlign = ac.indirectAlign.value();
@@ -1837,11 +2104,12 @@ CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp,
continue;
}
- // byval hands the callee a copy taken at the call, so naming the
- // object's own storage leaves it with the same copy and saves this
- // one. That holds only while nothing writes that storage between the
- // load and the call, since the operand carries the value as of the
- // load.
+ // byval hands the callee a copy taken at the call, so naming the object's
+ // own storage leaves it with the same copy and saves this one. That
+ // holds only while nothing writes that storage between the load and the
+ // call, since the operand carries the value as of the load. A volatile
+ // load is not forwarded, since the copy byval takes at the call is not
+ // volatile.
if (cir::LoadOp fwdLoad = forwardableIndirectLoad(srcLoad, minAlign);
fwdLoad && storageUnwrittenBetween(fwdLoad, call.getOperation(),
aliasAnalysis)) {
@@ -1852,6 +2120,10 @@ CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp,
// Otherwise fill a slot, at the load's position when there is one so
// the copy reads the memory state the load read.
+ if (failed(rejectAtomicRecordLoad(call, arg)))
+ return mlir::failure();
+ if (!srcLoad)
+ srcLoad = maybeGetCopyableRecordLoad(arg);
auto ptrTy = cir::PointerType::get(arg.getType());
mlir::IntegerAttr slotAlign = builder.getI64IntegerAttr(minAlign);
mlir::OpBuilder::InsertionGuard guard(builder);
@@ -1862,10 +2134,15 @@ CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp,
cir::AllocaOp::create(builder, call.getLoc(), ptrTy,
builder.getStringAttr("byval"), slotAlign);
if (srcLoad) {
- cir::CopyOp::create(builder, call.getLoc(), slot, srcLoad.getAddr(),
- slotAlign, loadSourceAlignment(srcLoad, builder));
+ auto copy = cir::CopyOp::create(builder, call.getLoc(), slot,
+ srcLoad.getAddr(), slotAlign,
+ loadSourceAlignment(srcLoad, builder));
+ copy.setIsVolatile(srcLoad.getIsVolatile());
deadRecordLoads.push_back(srcLoad);
} else {
+ // A record with no storage behind it, such as a constant, defines no
+ // bytes beyond its fields. A producer rewritten later turns this store
+ // into a copy through rewireRecordUses.
cir::StoreOp::create(builder, call.getLoc(), arg, slot);
}
newArgs.push_back(slot);
@@ -1885,8 +2162,10 @@ CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp,
// through a prepended pointer slot, not as a result), so dispatch to a
// dedicated helper for it; everything below handles the by-value returns.
if (fc.returnInfo.kind == ArgKind::Indirect && hasResult) {
- rewriteIndirectReturnCall(call, fc, newArgs, numEllipsisArgs, origRetTy,
- origCallArgTypes, builder, dl);
+ if (failed(rewriteIndirectReturnCall(call, fc, newArgs, numEllipsisArgs,
+ origRetTy, origCallArgTypes, builder,
+ dl)))
+ return mlir::failure();
eraseDeadRecordLoads(deadRecordLoads);
return mlir::success();
}
@@ -1906,14 +2185,17 @@ CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp,
call.getCalleeAttr(), callRetTy, newArgs);
copyCallAttributes(call, newCall);
- // Direct return with coercion: the new call returns the coerced type;
- // emit a coercion back to the original type for the call's existing uses.
+ // Direct return with coercion: the new call returns the coerced type, which
+ // is stored into a coercion slot where the old call's uses find the original
+ // type.
if (returnNeedsCoercion) {
builder.setInsertionPointAfter(newCall);
- mlir::Value coercedBack =
- emitCoercion(builder, call.getLoc(), origRetTy, newCall.getResult(),
- slotBlock, dl, fc.returnInfo.directOffset);
- call.getResult().replaceAllUsesWith(coercedBack);
+ SmallPtrSet<mlir::Operation *, 4> coercionOps;
+ if (failed(emitCoercionAndReplaceUses(builder, call.getLoc(), origRetTy,
+ newCall.getResult(), call.getResult(),
+ slotBlock, dl, coercionOps,
+ fc.returnInfo.directOffset)))
+ return mlir::failure();
}
if (needsArgAttrUpdate(fc)) {
@@ -2215,9 +2497,14 @@ mlir::Value copyRegisterToTemp(CIRBaseBuilderTy &b, mlir::Location loc,
return b.createPtrBitcast(temp, byteTy);
}
- mlir::Value val =
- b.createAlignedLoad(loc, b.createPtrBitcast(regAddr, f.resultTy), 8);
- b.createStore(loc, val, temp);
+ mlir::Value regView = b.createPtrBitcast(regAddr, f.resultTy);
+ if (mlir::isa<cir::RecordType>(f.resultTy))
+ cir::CopyOp::create(
+ b, loc, temp, regView,
+ b.getI64IntegerAttr(argumentAreaAlign(f.resultTy, f.module, f.dl)),
+ b.getI64IntegerAttr(8));
+ else
+ b.createStore(loc, b.createAlignedLoad(loc, regView, 8), temp);
return b.createPtrBitcast(temp, byteTy);
}
@@ -2373,10 +2660,17 @@ CIRABIRewriteContext::rewriteVAArg(mlir::Operation *vaArgOp,
}
// Every path above places the result at least this well aligned.
- mlir::Value result =
- builder.createAlignedLoad(loc, builder.createPtrBitcast(addr, resultTy),
- argumentAreaAlign(resultTy, module, dl));
- op.getResult().replaceAllUsesWith(result);
+ mlir::Value resultAddr = builder.createPtrBitcast(addr, resultTy);
+ uint64_t resultAlign = argumentAreaAlign(resultTy, module, dl);
+ if (mlir::isa<cir::RecordType>(resultTy)) {
+ SmallPtrSet<mlir::Operation *, 1> noExceptions;
+ if (failed(rewireRecordUses(builder, loc, op.getResult(), resultAddr,
+ resultAlign, dl, noExceptions)))
+ return mlir::failure();
+ } else {
+ op.getResult().replaceAllUsesWith(
+ builder.createAlignedLoad(loc, resultAddr, resultAlign));
+ }
op->erase();
return mlir::success();
}
diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h
index 85513762fc44a5..fe820bd43bd676 100644
--- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h
+++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.h
@@ -25,6 +25,7 @@
#include "mlir/IR/BuiltinOps.h"
#include "mlir/Interfaces/DataLayoutInterfaces.h"
#include "clang/CIR/Dialect/Analysis/CIRAliasAnalysis.h"
+#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
#include "clang/CIR/Dialect/IR/CIRDialect.h"
#include "llvm/ADT/SmallVector.h"
#include <cassert>
@@ -43,7 +44,7 @@ namespace cir {
class CIRABIRewriteContext : public mlir::abi::ABIRewriteContext {
public:
CIRABIRewriteContext(mlir::ModuleOp module, const mlir::DataLayout &dl)
- : module(module), dl(dl), aliasAnalysis(module) {
+ : module(module), dl(dl), cirDataLayout(module), aliasAnalysis(module) {
cir::registerCIRAliasAnalyses(aliasAnalysis, module);
}
@@ -115,6 +116,10 @@ class CIRABIRewriteContext : public mlir::abi::ABIRewriteContext {
mlir::ModuleOp module;
const mlir::DataLayout &dl;
+ /// The CIR view of the module's data layout, which provides the store and
+ /// alloc sizes that \c dl does not.
+ cir::CIRDataLayout cirDataLayout;
+
/// Alias analysis over the module with the CIR implementations registered.
/// rewriteCallSite asks it whether a byval operand's storage is written
/// between its load and the call.
diff --git a/clang/test/CIR/CodeGen/atomic.c b/clang/test/CIR/CodeGen/atomic.c
index 3481e6a56a8424..97e7435d68f7af 100644
--- a/clang/test/CIR/CodeGen/atomic.c
+++ b/clang/test/CIR/CodeGen/atomic.c
@@ -3978,11 +3978,14 @@ void store_atomic_different_size(S a) {
_Atomic(S) b;
__c11_atomic_store(&b, a, __ATOMIC_SEQ_CST);
+ // CIR: %[[COERCE:.*]] = cir.alloca "coerce" align(4) : !cir.ptr<!cir.int<u, 24>>
+ // CIR: cir.store %{{.+}}, %[[COERCE]] : !cir.int<u, 24>, !cir.ptr<!cir.int<u, 24>>
+ // CIR: %[[COERCE_S:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!cir.int<u, 24>> -> !cir.ptr<!rec_S>
// CIR: %[[A_ADDR:.*]] = cir.alloca "a" {{.*}} init : !cir.ptr<!rec_S>
// CIR: %[[B_ADDR:.*]] = cir.alloca "b" {{.*}} : !cir.ptr<!rec_anon_struct2>
// CIR: %[[A_ATOMIC_TMP_ADDR:.*]] = cir.alloca ".atomictmp" {{.*}} : !cir.ptr<!rec_S>
// CIR: %[[ATOMIC_TMP_ADDR:.*]] = cir.alloca "atomic-temp" {{.*}} : !cir.ptr<!rec_anon_struct2>
- // CIR: cir.store %[[A:.*]], %[[A_ADDR]] : !rec_S, !cir.ptr<!rec_S>
+ // CIR: cir.copy %[[COERCE_S]] align(4) to %[[A_ADDR]] align(1) : !cir.ptr<!rec_S>
// CIR: cir.copy %[[A_ADDR]] {{.*}} to %[[A_ATOMIC_TMP_ADDR]] {{.*}} : !cir.ptr<!rec_S>
// CIR: %[[B_VOID_PTR:.*]] = cir.cast bitcast %[[B_ADDR]] : !cir.ptr<!rec_anon_struct2> -> !cir.ptr<!u32i>
// CIR: %[[CONST_0:.*]] = cir.const #cir.int<0> : !u8i
@@ -3998,12 +4001,11 @@ void store_atomic_different_size(S a) {
// LLVM: %[[COERCE:.*]] = alloca i24, align 4
// LLVM: store i24 %{{.+}}, ptr %[[COERCE]], align 4
- // LLVM: %[[A:.*]] = load %struct.S, ptr %[[COERCE]], align 1
// LLVM: %[[A_ADDR:.*]] = alloca %struct.S, align 1
// LLVM: %[[B_ADDR:.*]] = alloca { %struct.S, [1 x i8] }, align 4
// LLVM: %[[A_ATOMIC_TMP_ADDR:.*]] = alloca %struct.S, align 1
// LLVM: %[[ATOMIC_TMP_ADDR:.*]] = alloca { %struct.S, [1 x i8] }, align 4
- // LLVM: store %struct.S %[[A]], ptr %[[A_ADDR]], align 1
+ // LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 1 %[[A_ADDR]], ptr align 4 %[[COERCE]], i64 3, i1 false)
// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 1 %[[A_ATOMIC_TMP_ADDR]], ptr align 1 %[[A_ADDR]], i64 3, i1 false)
// LLVM: call void @llvm.memset.p0.i64(ptr align 4 %[[ATOMIC_TMP_ADDR]], i8 0, i64 4, i1 false)
// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[ATOMIC_TMP_ADDR]], ptr align 1 %[[A_ATOMIC_TMP_ADDR]], i64 3, i1 false)
diff --git a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-empty.cpp b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-empty.cpp
index 1527cb19807314..e2716dea58655c 100644
--- a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-empty.cpp
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-empty.cpp
@@ -227,15 +227,18 @@ long callerHiWord(long hi) {
}
// CIR: cir.func {{.*}}@_Z12callerHiWordl(%arg0: !s64i {{.*}}) -> (!s64i
+// CIR: %[[COERCE:.+]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_HiWord>
+// CIR: %[[TMP:.+]] = cir.alloca "agg.tmp0" align(8) : !cir.ptr<!rec_HiWord>
// CIR: %[[RET:.+]] = cir.call @_Z10giveHiWordl(
// CIR: %[[ROFF:.+]] = cir.const #cir.int<8> : !s64i
// CIR: %[[RGEP:.+]] = cir.ptr_stride %{{.+}}, %[[ROFF]]
// CIR: %[[RPTR:.+]] = cir.cast bitcast %[[RGEP]] : !cir.ptr<!u8i> -> !cir.ptr<!s64i>
// CIR: cir.store %[[RET]], %[[RPTR]] : !s64i, !cir.ptr<!s64i>
+// CIR: cir.copy %[[COERCE]] align(8) to %[[TMP]] align(8) : !cir.ptr<!rec_HiWord>
// CIR: %[[AOFF:.+]] = cir.const #cir.int<8> : !s64i
// CIR: %[[AGEP:.+]] = cir.ptr_stride %{{.+}}, %[[AOFF]]
// CIR: %[[APTR:.+]] = cir.cast bitcast %[[AGEP]] : !cir.ptr<!u8i> -> !cir.ptr<!s64i>
-// CIR: %[[AVAL:.+]] = cir.load %[[APTR]] : !cir.ptr<!s64i>, !s64i
+// CIR: %[[AVAL:.+]] = cir.load align(8) %[[APTR]] : !cir.ptr<!s64i>, !s64i
// CIR: %{{.+}} = cir.call @_Z10takeHiWord6HiWord(%[[AVAL]])
// LLVM: define dso_local noundef i64 @_Z12callerHiWordl(i64 noundef %{{[^,)]+}})
// LLVM: %[[RET:.+]] = call i64 @_Z10giveHiWordl(i64 noundef %{{.+}})
diff --git a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-tail-padding.cpp b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-tail-padding.cpp
new file mode 100644
index 00000000000000..b8c19f88080bbd
--- /dev/null
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-tail-padding.cpp
@@ -0,0 +1,90 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefixes=LLVM,LLVMCIR --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefixes=LLVM,OGCG --input-file=%t.ll %s
+
+struct V { char v; };
+struct B { int i; char c; B(int); B() = default; };
+B makeB();
+
+struct C : virtual V {
+ [[no_unique_address]] B b;
+ C();
+};
+
+// b is at offset 8 with data size 5, and V is in its tail padding at 13.
+C::C() : b(makeB()) {}
+
+// CIR-LABEL: cir.func {{.*}} @_ZN1CC2Ev(
+// CIR: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+// CIR: %[[B:.*]] = cir.get_member %{{.*}}[1] {name = "b"}
+// CIR: %[[BPTR:.*]] = cir.cast bitcast %[[B]] : !cir.ptr<!rec_B2Ebase> -> !cir.ptr<!rec_B>
+// CIR: %[[RES:.*]] = cir.call @_Z5makeBv() : () -> !u64i
+// CIR: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+// CIR: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_B>
+// CIR: cir.copy %[[VIEW]] align(8) to %[[BPTR]] align(8) skip_tail_padding : !cir.ptr<!rec_B>
+
+// LLVM-LABEL: define dso_local void @_ZN1CC2Ev(ptr noundef nonnull align 8 dereferenceable(13) %{{[^,]+}}, ptr noundef %{{[^,)]+}})
+// LLVMCIR: %[[SLOT:.*]] = alloca i64, align 8
+// LLVM: %[[B:.*]] = getelementptr inbounds nuw %struct.C, ptr %{{[^,]+}}, i32 0, i32 1
+// LLVM: %[[RES:.*]] = call i64 @_Z5makeBv()
+// LLVMCIR: store i64 %[[RES]], ptr %[[SLOT]], align 8
+// LLVMCIR: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[B]], ptr align 8 %[[SLOT]], i64 5, i1 false)
+// OGCG: %[[TRUNC:.*]] = trunc i64 %[[RES]] to i40
+// OGCG: store i40 %[[TRUNC]], ptr %[[B]], align 8
+
+void localB() { B b = makeB(); }
+
+// CIR-LABEL: cir.func {{.*}} @_Z6localBv()
+// CIR: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+// CIR: %[[B:.*]] = cir.alloca "b" align(4) init : !cir.ptr<!rec_B>
+// CIR: %[[RES:.*]] = cir.call @_Z5makeBv() : () -> !u64i
+// CIR: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+// CIR: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_B>
+// CIR: cir.copy %[[VIEW]] align(8) to %[[B]] align(4) : !cir.ptr<!rec_B>
+
+// LLVM-LABEL: define dso_local void @_Z6localBv()
+// LLVMCIR: %[[SLOT:.*]] = alloca i64, align 8
+// LLVM: %[[B:.*]] = alloca %struct.B, align 4
+// LLVM: %[[RES:.*]] = call i64 @_Z5makeBv()
+// LLVMCIR: store i64 %[[RES]], ptr %[[SLOT]], align 8
+// LLVMCIR: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[B]], ptr align 8 %[[SLOT]], i64 8, i1 false)
+// OGCG: %[[TRUNC:.*]] = trunc i64 %[[RES]] to i40
+// OGCG: store i40 %[[TRUNC]], ptr %[[B]], align 4
+
+struct S { int a, b; };
+S makeS();
+
+void localS() { S s = makeS(); }
+
+// CIR-LABEL: cir.func {{.*}} @_Z6localSv()
+// CIR: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+// CIR: %[[S:.*]] = cir.alloca "s" align(4) init : !cir.ptr<!rec_S>
+// CIR: %[[RES:.*]] = cir.call @_Z5makeSv() : () -> !u64i
+// CIR: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+// CIR: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_S>
+// CIR: cir.copy %[[VIEW]] align(8) to %[[S]] align(4) : !cir.ptr<!rec_S>
+
+// LLVM-LABEL: define dso_local void @_Z6localSv()
+// LLVMCIR: %[[SLOT:.*]] = alloca i64, align 8
+// LLVM: %[[S:.*]] = alloca %struct.S, align 4
+// LLVM: %[[RES:.*]] = call i64 @_Z5makeSv()
+// LLVMCIR: store i64 %[[RES]], ptr %[[SLOT]], align 8
+// LLVMCIR: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[S]], ptr align 8 %[[SLOT]], i64 8, i1 false)
+// OGCG: store i64 %[[RES]], ptr %[[S]], align 4
+
+struct Big { long a, b, c; char d; };
+
+void takeBig(Big x) {}
+
+// CIR-LABEL: cir.func {{.*}} @_Z7takeBig3Big(%arg0: !cir.ptr<!rec_Big> {llvm.align = 8 : i64, llvm.byval = !rec_Big, llvm.noundef}
+// CIR: %[[X:.*]] = cir.alloca "x" align(8) init : !cir.ptr<!rec_Big>
+// CIR: cir.copy %arg0 align(8) to %[[X]] align(8) : !cir.ptr<!rec_Big>
+
+// LLVM-LABEL: define dso_local void @_Z7takeBig3Big(ptr noundef byval(%struct.Big) align 8 %{{[^,)]+}})
+// LLVMCIR: %[[X:.*]] = alloca %struct.Big, align 8
+// LLVMCIR: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[X]], ptr align 8 %{{[^,]+}}, i64 32, i1 false)
+// OGCG-NOT: call void @llvm.memcpy
+// LLVM: ret void
diff --git a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-union-copy.c b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-union-copy.c
new file mode 100644
index 00000000000000..599844229c565c
--- /dev/null
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-union-copy.c
@@ -0,0 +1,641 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefixes=LLVM,LLVMCIR --input-file=%t-cir.ll %s
+// RUN: FileCheck --check-prefix=NOMOVE --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefixes=LLVM,OGCG --input-file=%t.ll %s
+// RUN: FileCheck --check-prefix=NOMOVE --input-file=%t.ll %s
+
+typedef long double ld1 __attribute__((vector_size(16)));
+
+union H {
+ struct {
+ char a;
+ int b;
+ } s;
+ int c[2];
+};
+
+union U {
+ long double ld;
+ __int128 i;
+};
+
+union U2 {
+ long double ld;
+ struct {
+ unsigned long n;
+ const char *p;
+ } v;
+};
+
+union V {
+ ld1 v;
+ long double ld;
+};
+
+union VI {
+ ld1 v;
+ long l;
+};
+
+union C {
+ _Complex long double c;
+ long double ld;
+};
+
+union M {
+ long double ld;
+ char buf[32];
+};
+
+struct S16 {
+ int a, b, c, d;
+};
+
+// CIR-DAG: !rec_H = !cir.union<"H" {data !rec_anon2E0, data !cir.array<!s32i x 2>}>
+// CIR-DAG: !rec_U = !cir.union<"U" {data !cir.long_double<!cir.f80>, data !s128i}>
+// CIR-DAG: !rec_U2 = !cir.union<"U2" {data !cir.long_double<!cir.f80>, data !rec_anon2E1}>
+// CIR-DAG: !rec_V = !cir.union<"V" {data !cir.vector<1 x !cir.long_double<!cir.f80>>, data !cir.long_double<!cir.f80>}>
+// CIR-DAG: !rec_VI = !cir.union<"VI" {data !cir.vector<1 x !cir.long_double<!cir.f80>>, data !s64i}>
+// CIR-DAG: !rec_C = !cir.union<"C" {data !cir.complex<!cir.long_double<!cir.f80>>, data !cir.long_double<!cir.f80>}>
+// CIR-DAG: !rec_M = !cir.union<"M" {data !cir.long_double<!cir.f80>, data !cir.array<!s8i x 32>}, padding = {!cir.array<!u8i x 16>}>
+// CIR-DAG: !rec_anon_struct = !cir.struct<{data !u64i, data !u64i}>
+// CIR-DAG: !rec_anon_struct1 = !cir.struct<{data !u64i, data !cir.double}>
+// CIR-DAG: !rec_S16 = !cir.struct<"S16" {data !s32i, data !s32i, data !s32i, data !s32i}>
+
+// LLVM-DAG: %union.H = type { %struct.anon{{(\.[0-9]+)?}} }
+// LLVM-DAG: %union.U = type { x86_fp80 }
+// LLVM-DAG: %union.U2 = type { x86_fp80 }
+// LLVM-DAG: %union.V = type { <1 x x86_fp80> }
+// LLVM-DAG: %union.VI = type { <1 x x86_fp80> }
+// LLVM-DAG: %union.C = type { { x86_fp80, x86_fp80 } }
+// LLVM-DAG: %union.M = type { x86_fp80, [16 x i8] }
+// LLVM-DAG: %struct.S16 = type { i32, i32, i32, i32 }
+
+// The storage type of H is a struct with padding after s.a.
+union H retH(const union H *p) { return *p; }
+
+// CIR-LABEL: cir.func {{.*}} @retH(%arg0: !cir.ptr<!rec_H> {llvm.noundef} loc({{[^)]+}})) -> !u64i
+// CIR: %[[RETVAL:.+]] = cir.alloca "__retval" align(4) : !cir.ptr<!rec_H>
+// CIR: cir.copy %{{[^ ]+}} align(4) to %[[RETVAL]] align(4) : !cir.ptr<!rec_H>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[RETVAL]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+// CIR: %[[RET:.+]] = cir.load align(4) %[[VIEW]] : !cir.ptr<!u64i>, !u64i
+// CIR: cir.return %[[RET]] : !u64i
+
+// LLVM-LABEL: define dso_local i64 @retH(ptr noundef %{{[^,)]+}})
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %{{[^,)]+}}, ptr align 4 %{{[^,)]+}}, i64 8, i1 false)
+// LLVM: %[[RET:.+]] = load i64, ptr %{{[^,)]+}}, align 4
+// LLVM: ret i64 %[[RET]]
+
+// NOMOVE-LABEL: define {{.*}}@retH(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+int takeH(union H h) { return h.c[0]; }
+
+// CIR-LABEL: cir.func {{.*}} @takeH(%arg0: !u64i loc({{[^)]+}})) -> !s32i
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+// CIR: cir.store %arg0, %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+// CIR: %[[H:.+]] = cir.alloca "h" align(4) init : !cir.ptr<!rec_H>
+// CIR: cir.copy %[[VIEW]] align(8) to %[[H]] align(4) : !cir.ptr<!rec_H>
+
+// LLVM-LABEL: define dso_local i32 @takeH(i64 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@takeH(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+int bindH(const union H *p) {
+ union H d = retH(p);
+ return d.c[0];
+}
+
+// CIR-LABEL: cir.func {{.*}} @bindH(%arg0: !cir.ptr<!rec_H> {llvm.noundef} loc({{[^)]+}})) -> !s32i
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+// CIR: %[[D:.+]] = cir.alloca "d" align(4) init : !cir.ptr<!rec_H>
+// CIR: %[[RES:.+]] = cir.call @retH(%{{[^)]+}}) : (!cir.ptr<!rec_H> {llvm.noundef}) -> !u64i
+// CIR: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+// CIR: cir.copy %[[VIEW]] align(8) to %[[D]] align(4) : !cir.ptr<!rec_H>
+
+// LLVM-LABEL: define dso_local i32 @bindH(ptr noundef %{{[^,)]+}})
+// LLVM: call i64 @retH(ptr noundef %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@bindH(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+void storeH(union H *q, const union H *p) { *q = retH(p); }
+
+// CIR-LABEL: cir.func {{.*}} @storeH(%arg0: !cir.ptr<!rec_H> {llvm.noundef} loc({{[^)]+}}), %arg1: !cir.ptr<!rec_H> {llvm.noundef} loc({{[^)]+}}))
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+// CIR: %[[Q:.+]] = cir.load deref align(8) %{{[^ ]+}} : !cir.ptr<!cir.ptr<!rec_H>>, !cir.ptr<!rec_H>
+// CIR: %[[RES:.+]] = cir.call @retH(%{{[^)]+}}) : (!cir.ptr<!rec_H> {llvm.noundef}) -> !u64i
+// CIR: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+// CIR: cir.copy %[[VIEW]] align(8) to %[[Q]] align(4) : !cir.ptr<!rec_H>
+
+// LLVM-LABEL: define dso_local void @storeH(ptr noundef %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVM: call i64 @retH(ptr noundef %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@storeH(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+int passH(const union H *p) { return takeH(*p); }
+
+// CIR-LABEL: cir.func {{.*}} @passH(%arg0: !cir.ptr<!rec_H> {llvm.noundef} loc({{[^)]+}})) -> !s32i
+// CIR: %[[TMP:.+]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_H>
+// CIR: cir.copy %{{[^ ]+}} align(4) to %[[TMP]] align(8) : !cir.ptr<!rec_H>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[TMP]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+// CIR: %[[ARG:.+]] = cir.load align(8) %[[VIEW]] : !cir.ptr<!u64i>, !u64i
+// CIR: cir.call @takeH(%[[ARG]]) : (!u64i) -> !s32i
+
+// LLVM-LABEL: define dso_local i32 @passH(ptr noundef %{{[^,)]+}})
+// LLVM: call i32 @takeH(i64 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@passH(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+union H vaH(int n, ...) {
+ __builtin_va_list ap;
+ __builtin_va_start(ap, n);
+ union H h = __builtin_va_arg(ap, union H);
+ __builtin_va_end(ap);
+ return h;
+}
+
+// CIR-LABEL: cir.func {{.*}} @vaH(%arg0: !s32i {llvm.noundef} loc({{[^)]+}}), ...) -> !u64i
+// CIR: %[[RETVAL:.+]] = cir.alloca "__retval" align(4) init : !cir.ptr<!rec_H>
+// CIR: %[[TMP:.+]] = cir.alloca "vaarg.tmp" align(4) : !cir.ptr<!rec_H>
+// CIR: %[[ADDR:.+]] = cir.ternary(
+// CIR: %[[SRC:.+]] = cir.cast bitcast %[[ADDR]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_H>
+// CIR: cir.copy %[[SRC]] align(4) to %[[TMP]] align(4) : !cir.ptr<!rec_H>
+// CIR: cir.copy %[[TMP]] align(4) to %[[RETVAL]] align(4) : !cir.ptr<!rec_H>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[RETVAL]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+// CIR: %[[RET:.+]] = cir.load align(4) %[[VIEW]] : !cir.ptr<!u64i>, !u64i
+// CIR: cir.return %[[RET]] : !u64i
+
+// LLVM-LABEL: define dso_local i64 @vaH(i32 noundef %{{[^,)]+}}, ...)
+// LLVM: icmp ule i32 %{{.+}}, 40
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %{{[^,)]+}}, ptr align 4 %{{[^,)]+}}, i64 8, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@vaH(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+// The storage type of U is a long double that stores 10 of its 16 bytes.
+union U retU(const union U *p) { return *p; }
+
+// CIR-LABEL: cir.func {{.*}} @retU(%arg0: !cir.ptr<!rec_U> {llvm.noundef} loc({{[^)]+}})) -> !s128i
+// CIR: %[[RETVAL:.+]] = cir.alloca "__retval" align(16) : !cir.ptr<!rec_U>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[RETVAL]] align(16) : !cir.ptr<!rec_U>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[RETVAL]] : !cir.ptr<!rec_U> -> !cir.ptr<!s128i>
+// CIR: %[[RET:.+]] = cir.load align(16) %[[VIEW]] : !cir.ptr<!s128i>, !s128i
+// CIR: cir.return %[[RET]] : !s128i
+
+// LLVMCIR-LABEL: define dso_local i128 @retU(ptr noundef %{{[^,)]+}})
+// OGCG-LABEL: define dso_local { i64, i64 } @retU(ptr noundef %{{[^,)]+}})
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 16, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@retU(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+__int128 takeU(union U u) { return u.i; }
+
+// CIR-LABEL: cir.func {{.*}} @takeU(%arg0: !s128i loc({{[^)]+}})) -> !s128i
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!s128i>
+// CIR: cir.store %arg0, %[[SLOT]] : !s128i, !cir.ptr<!s128i>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!s128i> -> !cir.ptr<!rec_U>
+// CIR: %[[U:.+]] = cir.alloca "u" align(16) init : !cir.ptr<!rec_U>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[U]] align(16) : !cir.ptr<!rec_U>
+
+// LLVMCIR-LABEL: define dso_local i128 @takeU(i128 %{{[^,)]+}})
+// OGCG-LABEL: define dso_local i128 @takeU(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@takeU(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+__int128 bindU(const union U *p) {
+ union U d = retU(p);
+ return d.i;
+}
+
+// CIR-LABEL: cir.func {{.*}} @bindU(%arg0: !cir.ptr<!rec_U> {llvm.noundef} loc({{[^)]+}})) -> !s128i
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!s128i>
+// CIR: %[[D:.+]] = cir.alloca "d" align(16) init : !cir.ptr<!rec_U>
+// CIR: %[[RES:.+]] = cir.call @retU(%{{[^)]+}}) : (!cir.ptr<!rec_U> {llvm.noundef}) -> !s128i
+// CIR: cir.store %[[RES]], %[[SLOT]] : !s128i, !cir.ptr<!s128i>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!s128i> -> !cir.ptr<!rec_U>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[D]] align(16) : !cir.ptr<!rec_U>
+
+// LLVM-LABEL: define dso_local i128 @bindU(ptr noundef %{{[^,)]+}})
+// LLVMCIR: call i128 @retU(ptr noundef %{{[^,)]+}})
+// OGCG: call { i64, i64 } @retU(ptr noundef %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@bindU(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+void storeU(union U *q, const union U *p) { *q = retU(p); }
+
+// CIR-LABEL: cir.func {{.*}} @storeU(%arg0: !cir.ptr<!rec_U> {llvm.noundef} loc({{[^)]+}}), %arg1: !cir.ptr<!rec_U> {llvm.noundef} loc({{[^)]+}}))
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!s128i>
+// CIR: %[[Q:.+]] = cir.load deref align(8) %{{[^ ]+}} : !cir.ptr<!cir.ptr<!rec_U>>, !cir.ptr<!rec_U>
+// CIR: %[[RES:.+]] = cir.call @retU(%{{[^)]+}}) : (!cir.ptr<!rec_U> {llvm.noundef}) -> !s128i
+// CIR: cir.store %[[RES]], %[[SLOT]] : !s128i, !cir.ptr<!s128i>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!s128i> -> !cir.ptr<!rec_U>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[Q]] align(16) : !cir.ptr<!rec_U>
+
+// LLVM-LABEL: define dso_local void @storeU(ptr noundef %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVMCIR: call i128 @retU(ptr noundef %{{[^,)]+}})
+// OGCG: call { i64, i64 } @retU(ptr noundef %{{[^,)]+}})
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 16, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@storeU(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+__int128 passU(const union U *p) { return takeU(*p); }
+
+// CIR-LABEL: cir.func {{.*}} @passU(%arg0: !cir.ptr<!rec_U> {llvm.noundef} loc({{[^)]+}})) -> !s128i
+// CIR: %[[TMP:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_U>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_U>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[TMP]] : !cir.ptr<!rec_U> -> !cir.ptr<!s128i>
+// CIR: %[[ARG:.+]] = cir.load align(16) %[[VIEW]] : !cir.ptr<!s128i>, !s128i
+// CIR: cir.call @takeU(%[[ARG]]) : (!s128i) -> !s128i
+
+// LLVM-LABEL: define dso_local i128 @passU(ptr noundef %{{[^,)]+}})
+// LLVMCIR: call i128 @takeU(i128 %{{[^,)]+}})
+// OGCG: call i128 @takeU(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@passU(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+union U vaU(int n, ...) {
+ __builtin_va_list ap;
+ __builtin_va_start(ap, n);
+ union U u = __builtin_va_arg(ap, union U);
+ __builtin_va_end(ap);
+ return u;
+}
+
+// CIR-LABEL: cir.func {{.*}} @vaU(%arg0: !s32i {llvm.noundef} loc({{[^)]+}}), ...) -> !s128i
+// CIR: %[[RETVAL:.+]] = cir.alloca "__retval" align(16) init : !cir.ptr<!rec_U>
+// CIR: %[[TMP:.+]] = cir.alloca "vaarg.tmp" align(16) : !cir.ptr<!rec_U>
+// CIR: %[[REG:.+]] = cir.alloca "vaarg.reg" align(16) : !cir.ptr<!rec_U>
+// CIR: %[[ADDR:.+]] = cir.ternary(
+// CIR: %[[REGSLOT:.+]] = cir.cast bitcast %{{[^ ]+}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_U>
+// CIR: cir.copy %[[REGSLOT]] align(8) to %[[REG]] align(16) : !cir.ptr<!rec_U>
+// CIR: %[[SRC:.+]] = cir.cast bitcast %[[ADDR]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_U>
+// CIR: cir.copy %[[SRC]] align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_U>
+// CIR: cir.copy %[[TMP]] align(16) to %[[RETVAL]] align(16) : !cir.ptr<!rec_U>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[RETVAL]] : !cir.ptr<!rec_U> -> !cir.ptr<!s128i>
+// CIR: %[[RET:.+]] = cir.load align(16) %[[VIEW]] : !cir.ptr<!s128i>, !s128i
+// CIR: cir.return %[[RET]] : !s128i
+
+// LLVMCIR-LABEL: define dso_local i128 @vaU(i32 noundef %{{[^,)]+}}, ...)
+// OGCG-LABEL: define dso_local { i64, i64 } @vaU(i32 noundef %{{[^,)]+}}, ...)
+// LLVM: icmp ule i32 %{{.+}}, 32
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 8 %{{[^,)]+}}, i64 16, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 16, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@vaU(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+// The storage type of U2 is a long double. The struct member spans all 16
+// bytes.
+union U2 retU2(const union U2 *p) { return *p; }
+
+// CIR-LABEL: cir.func {{.*}} @retU2(%arg0: !cir.ptr<!rec_U2> {llvm.noundef} loc({{[^)]+}})) -> !rec_anon_struct
+// CIR: %[[RETVAL:.+]] = cir.alloca "__retval" align(16) : !cir.ptr<!rec_U2>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[RETVAL]] align(16) : !cir.ptr<!rec_U2>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[RETVAL]] : !cir.ptr<!rec_U2> -> !cir.ptr<!rec_anon_struct>
+// CIR: %[[RET:.+]] = cir.load align(16) %[[VIEW]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
+// CIR: cir.return %[[RET]] : !rec_anon_struct
+
+// LLVM-LABEL: define dso_local { i64, i64 } @retU2(ptr noundef %{{[^,)]+}})
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 16, i1 false)
+// LLVM: %[[RET:.+]] = load { i64, i64 }, ptr %{{[^,)]+}}, align 16
+// LLVM: ret { i64, i64 } %[[RET]]
+
+// NOMOVE-LABEL: define {{.*}}@retU2(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+const char *takeU2(union U2 u) { return u.v.p; }
+
+// CIR-LABEL: cir.func {{.*}} @takeU2(%arg0: !u64i loc({{[^)]+}}), %arg1: !u64i loc({{[^)]+}})) -> !cir.ptr<!s8i>
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_anon_struct>
+// CIR: %[[LOP:.+]] = cir.get_member %[[SLOT]][0] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: cir.store %arg0, %[[LOP]] : !u64i, !cir.ptr<!u64i>
+// CIR: %[[HIP:.+]] = cir.get_member %[[SLOT]][1] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: cir.store %arg1, %[[HIP]] : !u64i, !cir.ptr<!u64i>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_U2>
+// CIR: %[[U:.+]] = cir.alloca "u" align(16) init : !cir.ptr<!rec_U2>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[U]] align(16) : !cir.ptr<!rec_U2>
+
+// LLVM-LABEL: define dso_local ptr @takeU2(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@takeU2(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+const char *bindU2(const union U2 *p) {
+ union U2 d = retU2(p);
+ return d.v.p;
+}
+
+// CIR-LABEL: cir.func {{.*}} @bindU2(%arg0: !cir.ptr<!rec_U2> {llvm.noundef} loc({{[^)]+}})) -> !cir.ptr<!s8i>
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_anon_struct>
+// CIR: %[[D:.+]] = cir.alloca "d" align(16) init : !cir.ptr<!rec_U2>
+// CIR: %[[RES:.+]] = cir.call @retU2(%{{[^)]+}}) : (!cir.ptr<!rec_U2> {llvm.noundef}) -> !rec_anon_struct
+// CIR: cir.store %[[RES]], %[[SLOT]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_U2>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[D]] align(16) : !cir.ptr<!rec_U2>
+
+// LLVM-LABEL: define dso_local ptr @bindU2(ptr noundef %{{[^,)]+}})
+// LLVM: call { i64, i64 } @retU2(ptr noundef %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@bindU2(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+void storeU2(union U2 *q, const union U2 *p) { *q = retU2(p); }
+
+// CIR-LABEL: cir.func {{.*}} @storeU2(%arg0: !cir.ptr<!rec_U2> {llvm.noundef} loc({{[^)]+}}), %arg1: !cir.ptr<!rec_U2> {llvm.noundef} loc({{[^)]+}}))
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_anon_struct>
+// CIR: %[[Q:.+]] = cir.load deref align(8) %{{[^ ]+}} : !cir.ptr<!cir.ptr<!rec_U2>>, !cir.ptr<!rec_U2>
+// CIR: %[[RES:.+]] = cir.call @retU2(%{{[^)]+}}) : (!cir.ptr<!rec_U2> {llvm.noundef}) -> !rec_anon_struct
+// CIR: cir.store %[[RES]], %[[SLOT]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_U2>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[Q]] align(16) : !cir.ptr<!rec_U2>
+
+// LLVM-LABEL: define dso_local void @storeU2(ptr noundef %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVM: call { i64, i64 } @retU2(ptr noundef %{{[^,)]+}})
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 16, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@storeU2(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+const char *passU2(const union U2 *p) { return takeU2(*p); }
+
+// CIR-LABEL: cir.func {{.*}} @passU2(%arg0: !cir.ptr<!rec_U2> {llvm.noundef} loc({{[^)]+}})) -> !cir.ptr<!s8i>
+// CIR: %[[TMP:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_U2>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_U2>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[TMP]] : !cir.ptr<!rec_U2> -> !cir.ptr<!rec_anon_struct>
+// CIR: %[[LOP:.+]] = cir.get_member %[[VIEW]][0] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: %[[LO:.+]] = cir.load %[[LOP]] : !cir.ptr<!u64i>, !u64i
+// CIR: %[[HIP:.+]] = cir.get_member %[[VIEW]][1] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: %[[HI:.+]] = cir.load %[[HIP]] : !cir.ptr<!u64i>, !u64i
+// CIR: cir.call @takeU2(%[[LO]], %[[HI]]) : (!u64i, !u64i) -> !cir.ptr<!s8i>
+
+// LLVM-LABEL: define dso_local ptr @passU2(ptr noundef %{{[^,)]+}})
+// LLVM: call ptr @takeU2(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@passU2(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+union U2 vaU2(int n, ...) {
+ __builtin_va_list ap;
+ __builtin_va_start(ap, n);
+ union U2 u = __builtin_va_arg(ap, union U2);
+ __builtin_va_end(ap);
+ return u;
+}
+
+// CIR-LABEL: cir.func {{.*}} @vaU2(%arg0: !s32i {llvm.noundef} loc({{[^)]+}}), ...) -> !rec_anon_struct
+// CIR: %[[RETVAL:.+]] = cir.alloca "__retval" align(16) init : !cir.ptr<!rec_U2>
+// CIR: %[[TMP:.+]] = cir.alloca "vaarg.tmp" align(16) : !cir.ptr<!rec_U2>
+// CIR: %[[REG:.+]] = cir.alloca "vaarg.reg" align(16) : !cir.ptr<!rec_U2>
+// CIR: %[[ADDR:.+]] = cir.ternary(
+// CIR: %[[REGSLOT:.+]] = cir.cast bitcast %{{[^ ]+}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_U2>
+// CIR: cir.copy %[[REGSLOT]] align(8) to %[[REG]] align(16) : !cir.ptr<!rec_U2>
+// CIR: %[[SRC:.+]] = cir.cast bitcast %[[ADDR]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_U2>
+// CIR: cir.copy %[[SRC]] align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_U2>
+// CIR: cir.copy %[[TMP]] align(16) to %[[RETVAL]] align(16) : !cir.ptr<!rec_U2>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[RETVAL]] : !cir.ptr<!rec_U2> -> !cir.ptr<!rec_anon_struct>
+// CIR: %[[RET:.+]] = cir.load align(16) %[[VIEW]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
+// CIR: cir.return %[[RET]] : !rec_anon_struct
+
+// LLVM-LABEL: define dso_local { i64, i64 } @vaU2(i32 noundef %{{[^,)]+}}, ...)
+// LLVM: icmp ule i32 %{{.+}}, 32
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 8 %{{[^,)]+}}, i64 16, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 16, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@vaU2(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+union V takeV(union V u) { return u; }
+
+// CIR-LABEL: cir.func {{.*}} @takeV(%arg0: !cir.ptr<!rec_V> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.noalias, llvm.sret = !rec_V, llvm.writable} loc({{[^)]+}}), %arg1: !cir.ptr<!rec_V> {llvm.align = 16 : i64, llvm.byval = !rec_V, llvm.noundef} loc({{[^)]+}}))
+// CIR: %[[U:.+]] = cir.alloca "u" align(16) init : !cir.ptr<!rec_V>
+// CIR: cir.copy %arg1 align(16) to %[[U]] align(16) : !cir.ptr<!rec_V>
+// CIR: cir.copy %[[U]] align(16) to %arg0 align(16) : !cir.ptr<!rec_V>
+
+// LLVM-LABEL: define dso_local void @takeV(ptr dead_on_unwind noalias writable sret(%union.V) align 16 %{{[^,)]+}}, ptr noundef byval(%union.V) align 16 %{{[^,)]+}})
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 16, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@takeV(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+union V passV(const union V *p) { return takeV(*p); }
+
+// CIR-LABEL: cir.func {{.*}} @passV(%arg0: !cir.ptr<!rec_V> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.noalias, llvm.sret = !rec_V, llvm.writable} loc({{[^)]+}}), %arg1: !cir.ptr<!rec_V> {llvm.noundef} loc({{[^)]+}}))
+// CIR: %[[BYVAL:.+]] = cir.alloca "byval" align(16) : !cir.ptr<!rec_V>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[BYVAL]] align(16) : !cir.ptr<!rec_V>
+// CIR: cir.call @takeV(%arg0, %[[BYVAL]]) : (!cir.ptr<!rec_V> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.sret = !rec_V, llvm.writable}, !cir.ptr<!rec_V> {llvm.align = 16 : i64, llvm.byval = !rec_V, llvm.noundef}) -> ()
+
+// LLVM-LABEL: define dso_local void @passV(ptr dead_on_unwind noalias writable sret(%union.V) align 16 %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVM: call void @takeV(ptr dead_on_unwind writable sret(%union.V) align 16 %{{[^,)]+}}, ptr noundef byval(%union.V) align 16 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@passV(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+union VI takeVI(union VI u) { return u; }
+
+// CIR-LABEL: cir.func {{.*}} @takeVI(%arg0: !u64i loc({{[^)]+}}), %arg1: !cir.double loc({{[^)]+}})) -> !rec_anon_struct1
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_anon_struct1>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_anon_struct1> -> !cir.ptr<!rec_VI>
+// CIR: %[[U:.+]] = cir.alloca "u" align(16) init : !cir.ptr<!rec_VI>
+// CIR: %[[RETVAL:.+]] = cir.alloca "__retval" align(16) : !cir.ptr<!rec_VI>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[U]] align(16) : !cir.ptr<!rec_VI>
+// CIR: cir.copy %[[U]] align(16) to %[[RETVAL]] align(16) : !cir.ptr<!rec_VI>
+// CIR: %[[RETVIEW:.+]] = cir.cast bitcast %[[RETVAL]] : !cir.ptr<!rec_VI> -> !cir.ptr<!rec_anon_struct1>
+// CIR: %[[RET:.+]] = cir.load align(16) %[[RETVIEW]] : !cir.ptr<!rec_anon_struct1>, !rec_anon_struct1
+// CIR: cir.return %[[RET]] : !rec_anon_struct1
+
+// LLVM-LABEL: define dso_local { i64, double } @takeVI(i64 %{{[^,)]+}}, double %{{[^,)]+}})
+// LLVM: %[[RET:.+]] = load { i64, double }, ptr %{{[^,)]+}}, align 16
+// LLVM: ret { i64, double } %[[RET]]
+
+// NOMOVE-LABEL: define {{.*}}@takeVI(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+union VI passVI(const union VI *p) { return takeVI(*p); }
+
+// CIR-LABEL: cir.func {{.*}} @passVI(%arg0: !cir.ptr<!rec_VI> {llvm.noundef} loc({{[^)]+}})) -> !rec_anon_struct1
+// CIR: %[[RSLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_anon_struct1>
+// CIR: %[[TMP:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_VI>
+// CIR: %[[RETVAL:.+]] = cir.alloca "__retval" align(16) : !cir.ptr<!rec_VI>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_VI>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[TMP]] : !cir.ptr<!rec_VI> -> !cir.ptr<!rec_anon_struct1>
+// CIR: %[[LOP:.+]] = cir.get_member %[[VIEW]][0] {name = ""} : !cir.ptr<!rec_anon_struct1> -> !cir.ptr<!u64i>
+// CIR: %[[LO:.+]] = cir.load %[[LOP]] : !cir.ptr<!u64i>, !u64i
+// CIR: %[[HIP:.+]] = cir.get_member %[[VIEW]][1] {name = ""} : !cir.ptr<!rec_anon_struct1> -> !cir.ptr<!cir.double>
+// CIR: %[[HI:.+]] = cir.load %[[HIP]] : !cir.ptr<!cir.double>, !cir.double
+// CIR: %[[RES:.+]] = cir.call @takeVI(%[[LO]], %[[HI]]) : (!u64i, !cir.double) -> !rec_anon_struct1
+// CIR: cir.store %[[RES]], %[[RSLOT]] : !rec_anon_struct1, !cir.ptr<!rec_anon_struct1>
+// CIR: %[[RVIEW:.+]] = cir.cast bitcast %[[RSLOT]] : !cir.ptr<!rec_anon_struct1> -> !cir.ptr<!rec_VI>
+// CIR: cir.copy %[[RVIEW]] align(16) to %[[RETVAL]] align(16) : !cir.ptr<!rec_VI>
+
+// LLVM-LABEL: define dso_local { i64, double } @passVI(ptr noundef %{{[^,)]+}})
+// LLVM: call { i64, double } @takeVI(i64 %{{[^,)]+}}, double %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@passVI(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+union C takeC(union C u) { return u; }
+
+// CIR-LABEL: cir.func {{.*}} @takeC(%arg0: !cir.ptr<!rec_C> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.noalias, llvm.sret = !rec_C, llvm.writable} loc({{[^)]+}}), %arg1: !cir.ptr<!rec_C> {llvm.align = 16 : i64, llvm.byval = !rec_C, llvm.noundef} loc({{[^)]+}}))
+// CIR: %[[U:.+]] = cir.alloca "u" align(16) init : !cir.ptr<!rec_C>
+// CIR: cir.copy %arg1 align(16) to %[[U]] align(16) : !cir.ptr<!rec_C>
+// CIR: cir.copy %[[U]] align(16) to %arg0 align(16) : !cir.ptr<!rec_C>
+
+// LLVM-LABEL: define dso_local void @takeC(ptr dead_on_unwind noalias writable sret(%union.C) align 16 %{{[^,)]+}}, ptr noundef byval(%union.C) align 16 %{{[^,)]+}})
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 32, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@takeC(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+union C passC(const union C *p) { return takeC(*p); }
+
+// CIR-LABEL: cir.func {{.*}} @passC(%arg0: !cir.ptr<!rec_C> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.noalias, llvm.sret = !rec_C, llvm.writable} loc({{[^)]+}}), %arg1: !cir.ptr<!rec_C> {llvm.noundef} loc({{[^)]+}}))
+// CIR: %[[BYVAL:.+]] = cir.alloca "byval" align(16) : !cir.ptr<!rec_C>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[BYVAL]] align(16) : !cir.ptr<!rec_C>
+// CIR: cir.call @takeC(%arg0, %[[BYVAL]]) : (!cir.ptr<!rec_C> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.sret = !rec_C, llvm.writable}, !cir.ptr<!rec_C> {llvm.align = 16 : i64, llvm.byval = !rec_C, llvm.noundef}) -> ()
+
+// LLVM-LABEL: define dso_local void @passC(ptr dead_on_unwind noalias writable sret(%union.C) align 16 %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVM: call void @takeC(ptr dead_on_unwind writable sret(%union.C) align 16 %{{[^,)]+}}, ptr noundef byval(%union.C) align 16 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@passC(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+// M is 32 bytes, so it is passed and returned in memory.
+union M retM(const union M *p) { return *p; }
+
+// CIR-LABEL: cir.func {{.*}} @retM(%arg0: !cir.ptr<!rec_M> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.noalias, llvm.sret = !rec_M, llvm.writable} loc({{[^)]+}}), %arg1: !cir.ptr<!rec_M> {llvm.noundef} loc({{[^)]+}}))
+// CIR: cir.copy %{{[^ ]+}} align(16) to %arg0 align(16) : !cir.ptr<!rec_M>
+
+// LLVM-LABEL: define dso_local void @retM(ptr dead_on_unwind noalias writable sret(%union.M) align 16 %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 32, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@retM(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+char takeM(union M m) { return m.buf[31]; }
+
+// CIR-LABEL: cir.func {{.*}} @takeM(%arg0: !cir.ptr<!rec_M> {llvm.align = 16 : i64, llvm.byval = !rec_M, llvm.noundef} loc({{[^)]+}})) -> (!s8i {llvm.signext})
+// CIR: %[[M:.+]] = cir.alloca "m" align(16) init : !cir.ptr<!rec_M>
+// CIR: cir.copy %arg0 align(16) to %[[M]] align(16) : !cir.ptr<!rec_M>
+
+// LLVM-LABEL: define dso_local signext i8 @takeM(ptr noundef byval(%union.M) align 16 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@takeM(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+void storeM(union M *q, const union M *p) { *q = retM(p); }
+
+// CIR-LABEL: cir.func {{.*}} @storeM(%arg0: !cir.ptr<!rec_M> {llvm.noundef} loc({{[^)]+}}), %arg1: !cir.ptr<!rec_M> {llvm.noundef} loc({{[^)]+}}))
+// CIR: cir.call @retM(%{{[^,]+}}, %{{[^)]+}}) : (!cir.ptr<!rec_M> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.sret = !rec_M, llvm.writable}, !cir.ptr<!rec_M> {llvm.noundef}) -> ()
+
+// LLVM-LABEL: define dso_local void @storeM(ptr noundef %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVM: call void @retM(ptr dead_on_unwind writable sret(%union.M) align 16 %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@storeM(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+char passM(const union M *p) { return takeM(*p); }
+
+// CIR-LABEL: cir.func {{.*}} @passM(%arg0: !cir.ptr<!rec_M> {llvm.noundef} loc({{[^)]+}})) -> (!s8i {llvm.signext})
+// CIR: %[[BYVAL:.+]] = cir.alloca "byval" align(16) : !cir.ptr<!rec_M>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[BYVAL]] align(16) : !cir.ptr<!rec_M>
+// CIR: cir.call @takeM(%[[BYVAL]]) : (!cir.ptr<!rec_M> {llvm.align = 16 : i64, llvm.byval = !rec_M, llvm.noundef}) -> (!s8i {llvm.signext})
+
+// LLVM-LABEL: define dso_local signext i8 @passM(ptr noundef %{{[^,)]+}})
+// LLVM: call signext i8 @takeM(ptr noundef byval(%union.M) align 16 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@passM(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+union M vaM(int n, ...) {
+ __builtin_va_list ap;
+ __builtin_va_start(ap, n);
+ union M m = __builtin_va_arg(ap, union M);
+ __builtin_va_end(ap);
+ return m;
+}
+
+// CIR-LABEL: cir.func {{.*}} @vaM(%arg0: !cir.ptr<!rec_M> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.noalias, llvm.sret = !rec_M, llvm.writable} loc({{[^)]+}}), %arg1: !s32i {llvm.noundef} loc({{[^)]+}}), ...)
+// CIR: %[[TMP:.+]] = cir.alloca "vaarg.tmp" align(16) : !cir.ptr<!rec_M>
+// CIR-NOT: gp_offset
+// CIR: %[[ALIGNED:.+]] = cir.ptr_mask %{{.+}}, %{{.+}} : (!cir.ptr<!u8i>, !s64i) -> !cir.ptr<!u8i>
+// CIR: %[[SRC:.+]] = cir.cast bitcast %[[ALIGNED]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_M>
+// CIR: cir.copy %[[SRC]] align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_M>
+// CIR: cir.copy %[[TMP]] align(16) to %arg0 align(16) : !cir.ptr<!rec_M>
+
+// LLVM-LABEL: define dso_local void @vaM(ptr dead_on_unwind noalias writable sret(%union.M) align 16 %{{[^,)]+}}, i32 noundef %{{[^,)]+}}, ...)
+// LLVM-NOT: icmp
+// LLVM: call ptr @llvm.ptrmask.p0.i64(ptr %{{.+}}, i64 -16)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 32, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@vaM(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+void takeS16(struct S16);
+
+// The argument is read in place from the align-4 local as two i64 eightbytes.
+void passS16(struct S16 *p) {
+ struct S16 s = *p;
+ takeS16(s);
+}
+
+// CIR-LABEL: cir.func {{.*}} @passS16(%arg0: !cir.ptr<!rec_S16> {llvm.noundef} loc({{[^)]+}}))
+// CIR: %[[S:.+]] = cir.alloca "s" align(4) init : !cir.ptr<!rec_S16>
+// CIR: cir.copy %{{[^ ]+}} align(4) to %[[S]] align(4) : !cir.ptr<!rec_S16>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[S]] : !cir.ptr<!rec_S16> -> !cir.ptr<!rec_anon_struct>
+// CIR: %[[LOP:.+]] = cir.get_member %[[VIEW]][0] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: %[[LO:.+]] = cir.load align(4) %[[LOP]] : !cir.ptr<!u64i>, !u64i
+// CIR: %[[HIP:.+]] = cir.get_member %[[VIEW]][1] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: %[[HI:.+]] = cir.load align(4) %[[HIP]] : !cir.ptr<!u64i>, !u64i
+// CIR: cir.call @takeS16(%[[LO]], %[[HI]]) : (!u64i, !u64i) -> ()
+
+// LLVM-LABEL: define dso_local void @passS16(ptr noundef %{{[^,)]+}})
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[S:[^,]+]], ptr align 4 %{{[^,)]+}}, i64 16, i1 false)
+// LLVM: %[[LOP:.+]] = getelementptr inbounds nuw { i64, i64 }, ptr %[[S]], i32 0, i32 0
+// LLVM: %[[LO:.+]] = load i64, ptr %[[LOP]], align 4
+// LLVM: %[[HIP:.+]] = getelementptr inbounds nuw { i64, i64 }, ptr %[[S]], i32 0, i32 1
+// LLVM: %[[HI:.+]] = load i64, ptr %[[HIP]], align 4
+// LLVM: call void @takeS16(i64 %[[LO]], i64 %[[HI]])
+
+// NOMOVE-LABEL: define {{.*}}@passS16(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+// An X87-class union result stored through a pointer is copied whole out of
+// the slot its x86_fp80 is stored into.
+union X87U { long double a, b; };
+union X87U makeX87U(void);
+
+void storeX87U(union X87U *q) { *q = makeX87U(); }
+
+// CIR-LABEL: cir.func {{.*}} @storeX87U(%arg0: !cir.ptr<!rec_X87U> {llvm.noundef} loc({{[^)]+}}))
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!cir.f80>
+// CIR: %[[Q:.+]] = cir.load deref align(8) %{{[^ ]+}} : !cir.ptr<!cir.ptr<!rec_X87U>>, !cir.ptr<!rec_X87U>
+// CIR: %[[RES:.+]] = cir.call @makeX87U() : () -> !cir.f80
+// CIR: cir.store %[[RES]], %[[SLOT]] : !cir.f80, !cir.ptr<!cir.f80>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!cir.f80> -> !cir.ptr<!rec_X87U>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[Q]] align(16) : !cir.ptr<!rec_X87U>
+
+// LLVM-LABEL: define dso_local void @storeX87U(ptr noundef %{{[^,)]+}})
+// LLVMCIR: %[[TMP:.+]] = alloca x86_fp80, align 16
+// OGCG: %[[TMP:.+]] = alloca %union.X87U, align 16
+// LLVM: %[[Q:.+]] = load ptr, ptr %{{[^,)]+}}, align 8
+// LLVM: %[[RES:.+]] = call x86_fp80 @makeX87U()
+// LLVM: store x86_fp80 %[[RES]], ptr %[[TMP]], align 16
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[Q]], ptr align 16 %[[TMP]], i64 16, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@storeX87U(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* %union\.}}
diff --git a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-variadic.c b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-variadic.c
index d5e17cf10a8d0c..5951886d05dd6d 100644
--- a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-variadic.c
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-variadic.c
@@ -20,7 +20,7 @@ int call_scalar(Pair2 p, int a, double d) { return vf(p, a, d); }
// CIR: cir.store %arg2, %[[DSLOT:[0-9]+]] : !cir.double, !cir.ptr<!cir.double>
// CIR: %[[AV:[0-9]+]] = cir.load align(4) %[[ASLOT]] : !cir.ptr<!s32i>, !s32i
// CIR: %[[DV:[0-9]+]] = cir.load align(8) %[[DSLOT]] : !cir.ptr<!cir.double>, !cir.double
-// CIR: %[[PV:[0-9]+]] = cir.load %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
+// CIR: %[[PV:[0-9]+]] = cir.load align(4) %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
// CIR: cir.call @vf(%[[PV]], %[[AV]], %[[DV]]) : (!u64i, !s32i {llvm.noundef}, !cir.double {llvm.noundef}) -> !s32i
// CIR: cir.func private @vf(!u64i, ...) -> !s32i
@@ -43,11 +43,12 @@ int call_small(Pair2 p, Pair16 q) { return vf(p, q); }
// CIR: cir.store %arg1, %[[IN0]] : !s64i, !cir.ptr<!s64i>
// CIR: %[[IN1:[0-9]+]] = cir.get_member %[[IN]][1] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!s64i>
// CIR: cir.store %arg2, %[[IN1]] : !s64i, !cir.ptr<!s64i>
-// CIR: %[[PV:[0-9]+]] = cir.load %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
-// CIR: %[[OUT0:[0-9]+]] = cir.get_member %[[OUT:[0-9]+]][0] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!s64i>
-// CIR: %[[Q0:[0-9]+]] = cir.load %[[OUT0]] : !cir.ptr<!s64i>, !s64i
+// CIR: %[[PV:[0-9]+]] = cir.load align(4) %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
+// CIR: %[[OUT:[0-9]+]] = cir.cast bitcast %{{[0-9]+}} : !cir.ptr<!rec_Pair16> -> !cir.ptr<!rec_anon_struct>
+// CIR: %[[OUT0:[0-9]+]] = cir.get_member %[[OUT]][0] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!s64i>
+// CIR: %[[Q0:[0-9]+]] = cir.load align(8) %[[OUT0]] : !cir.ptr<!s64i>, !s64i
// CIR: %[[OUT1:[0-9]+]] = cir.get_member %[[OUT]][1] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!s64i>
-// CIR: %[[Q1:[0-9]+]] = cir.load %[[OUT1]] : !cir.ptr<!s64i>, !s64i
+// CIR: %[[Q1:[0-9]+]] = cir.load align(8) %[[OUT1]] : !cir.ptr<!s64i>, !s64i
// CIR: cir.call @vf(%[[PV]], %[[Q0]], %[[Q1]]) : (!u64i, !s64i, !s64i) -> !s32i
// LLVM-LABEL: define dso_local i32 @call_small(
@@ -70,21 +71,16 @@ int call_big(Pair2 p, Big b) { return vf(p, b); }
// CIR: %[[B:[0-9]+]] = cir.alloca "b" align(8) init : !cir.ptr<!rec_Big>
// CIR-NOT: cir.alloca "byval"
// CIR: cir.copy %arg1 align(8) to %[[B]] align(8) : !cir.ptr<!rec_Big>
-// CIR: %[[PV:[0-9]+]] = cir.load %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
+// CIR: %[[PV:[0-9]+]] = cir.load align(4) %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
// CIR-NEXT: %{{[0-9]+}} = cir.call @vf(%[[PV]], %[[B]]) : (!u64i, !cir.ptr<!rec_Big> {llvm.align = 8 : i64, llvm.byval = !rec_Big, llvm.noundef}) -> !s32i
-// The parameter's own slot is the operand's storage, so the call forwards it
-// rather than building a second one. What is left between CIR and the
-// reference is that slot and the Pair2 coerce round trip, at CIR's own
-// alignment.
// LLVM-LABEL: define dso_local i32 @call_big(
// LLVM-SAME: i64 %[[P:[0-9a-zA-Z._]+]], ptr noundef byval(%struct.Big) align 8 %[[B:[0-9a-zA-Z._]+]])
// LLVM-CIR: %[[BSLOT:[0-9a-zA-Z._]+]] = alloca %struct.Big, align 8
// LLVM-CIR: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[BSLOT]], ptr align 8 %[[B]], i64 32, i1 false)
-// LLVM-CIR: %[[PV:[0-9a-zA-Z._]+]] = load i64, ptr %{{[0-9a-zA-Z._]+}}, align 8
-// LLVM-CIR-NEXT: %{{[0-9a-zA-Z._]+}} = call i32 (i64, ...) @vf(i64 %[[PV]], ptr noundef byval(%struct.Big) align 8 %[[BSLOT]])
// LLVM-OGCG-NOT: memcpy
-// LLVM-OGCG: %[[PV:[0-9a-zA-Z._]+]] = load i64, ptr %{{[0-9a-zA-Z._]+}}, align 4
+// LLVM: %[[PV:[0-9a-zA-Z._]+]] = load i64, ptr %{{[0-9a-zA-Z._]+}}, align 4
+// LLVM-CIR-NEXT: %{{[0-9a-zA-Z._]+}} = call i32 (i64, ...) @vf(i64 %[[PV]], ptr noundef byval(%struct.Big) align 8 %[[BSLOT]])
// LLVM-OGCG-NEXT: %{{[0-9a-zA-Z._]+}} = call i32 (i64, ...) @vf(i64 %[[PV]], ptr noundef byval(%struct.Big) align 8 %[[B]])
// The same Pair16 that went to registers in call_small goes to memory here:
@@ -106,13 +102,11 @@ int call_exhausted(Pair2 p, long a, long b, long c, long d, Pair16 q) {
// CIR: %[[CV:[0-9]+]] = cir.load align(8) %[[CS]] : !cir.ptr<!s64i>, !s64i
// CIR: %[[DV:[0-9]+]] = cir.load align(8) %[[DS]] : !cir.ptr<!s64i>, !s64i
// CIR-NOT: cir.alloca "byval"
-// CIR: %[[PV:[0-9]+]] = cir.load %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
+// CIR: %[[PV:[0-9]+]] = cir.load align(4) %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
// CIR-NEXT: %{{[0-9]+}} = cir.call @vf(%[[PV]], %[[AV]], %[[BV]], %[[CV]], %[[DV]], %[[Q]]) : (!u64i, !s64i {llvm.noundef}, !s64i {llvm.noundef}, !s64i {llvm.noundef}, !s64i {llvm.noundef}, !cir.ptr<!rec_Pair16> {llvm.align = 8 : i64, llvm.byval = !rec_Pair16, llvm.noundef}) -> !s32i
-// LLVM-CIR-LABEL: define dso_local i32 @call_exhausted(
-// LLVM-CIR-SAME: i64 %[[P:[0-9a-zA-Z._]+]], i64 noundef %[[A:[0-9a-zA-Z._]+]], i64 noundef %[[B:[0-9a-zA-Z._]+]], i64 noundef %[[C:[0-9a-zA-Z._]+]], i64 noundef %[[D:[0-9a-zA-Z._]+]], ptr noundef byval(%struct.Pair16) align 8 %[[Q:[0-9a-zA-Z._]+]])
-// LLVM-OGCG-LABEL: define dso_local i32 @call_exhausted(
-// LLVM-OGCG-SAME: i64 %[[P:[0-9a-zA-Z._]+]], i64 noundef %[[A:[0-9a-zA-Z._]+]], i64 noundef %[[B:[0-9a-zA-Z._]+]], i64 noundef %[[C:[0-9a-zA-Z._]+]], i64 noundef %[[D:[0-9a-zA-Z._]+]], ptr noundef byval(%struct.Pair16) align 8 %[[Q:[0-9a-zA-Z._]+]])
+// LLVM-LABEL: define dso_local i32 @call_exhausted(
+// LLVM-SAME: i64 %[[P:[0-9a-zA-Z._]+]], i64 noundef %[[A:[0-9a-zA-Z._]+]], i64 noundef %[[B:[0-9a-zA-Z._]+]], i64 noundef %[[C:[0-9a-zA-Z._]+]], i64 noundef %[[D:[0-9a-zA-Z._]+]], ptr noundef byval(%struct.Pair16) align 8 %[[Q:[0-9a-zA-Z._]+]])
// LLVM: store i64 %[[A]], ptr %[[AS:[0-9a-zA-Z._]+]], align 8
// LLVM: store i64 %[[B]], ptr %[[BS:[0-9a-zA-Z._]+]], align 8
// LLVM: store i64 %[[C]], ptr %[[CS:[0-9a-zA-Z._]+]], align 8
@@ -123,9 +117,8 @@ int call_exhausted(Pair2 p, long a, long b, long c, long d, Pair16 q) {
// LLVM: %[[BV:[0-9a-zA-Z._]+]] = load i64, ptr %[[BS]], align 8
// LLVM: %[[CV:[0-9a-zA-Z._]+]] = load i64, ptr %[[CS]], align 8
// LLVM: %[[DV:[0-9a-zA-Z._]+]] = load i64, ptr %[[DS]], align 8
-// LLVM-CIR: %[[PV:[0-9a-zA-Z._]+]] = load i64, ptr %{{[0-9a-zA-Z._]+}}, align 8
+// LLVM: %[[PV:[0-9a-zA-Z._]+]] = load i64, ptr %{{[0-9a-zA-Z._]+}}, align 4
// LLVM-CIR-NEXT: %{{[0-9a-zA-Z._]+}} = call i32 (i64, ...) @vf(i64 %[[PV]], i64 noundef %[[AV]], i64 noundef %[[BV]], i64 noundef %[[CV]], i64 noundef %[[DV]], ptr noundef byval(%struct.Pair16) align 8 %[[QSLOT]])
-// LLVM-OGCG: %[[PV:[0-9a-zA-Z._]+]] = load i64, ptr %{{[0-9a-zA-Z._]+}}, align 4
// LLVM-OGCG-NEXT: %{{[0-9a-zA-Z._]+}} = call i32 (i64, ...) @vf(i64 %[[PV]], i64 noundef %[[AV]], i64 noundef %[[BV]], i64 noundef %[[CV]], i64 noundef %[[DV]], ptr noundef byval(%struct.Pair16) align 8 %[[Q]])
// A 128-bit integer spans two eightbytes but is still passed whole.
@@ -134,7 +127,7 @@ int call_int128(Pair2 p, __int128 w) { return vf(p, w); }
// CIR-LABEL: cir.func {{.*}}@call_int128(%arg0: !u64i loc({{.+}}), %arg1: !s128i {llvm.noundef} loc({{.+}})) -> !s32i
// CIR: cir.store %arg1, %[[WSLOT:[0-9]+]] : !s128i, !cir.ptr<!s128i>
// CIR: %[[WV:[0-9]+]] = cir.load align(16) %[[WSLOT]] : !cir.ptr<!s128i>, !s128i
-// CIR: %[[PV:[0-9]+]] = cir.load %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
+// CIR: %[[PV:[0-9]+]] = cir.load align(4) %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
// CIR: cir.call @vf(%[[PV]], %[[WV]]) : (!u64i, !s128i {llvm.noundef}) -> !s32i
// LLVM-LABEL: define dso_local i32 @call_int128(
@@ -151,20 +144,19 @@ int call_wide(Pair2 p, Wide w) { return vf(p, w); }
// CIR-LABEL: cir.func {{.*}}@call_wide(%arg0: !u64i loc({{.+}}), %arg1: !s128i loc({{.+}})) -> !s32i
// CIR: cir.store %arg1, %[[IN:[0-9]+]] : !s128i, !cir.ptr<!s128i>
// CIR: %[[INCAST:[0-9]+]] = cir.cast bitcast %[[IN]] : !cir.ptr<!s128i> -> !cir.ptr<!rec_Wide>
-// CIR: %{{[0-9]+}} = cir.load %[[INCAST]] : !cir.ptr<!rec_Wide>, !rec_Wide
-// CIR: %[[PV:[0-9]+]] = cir.load %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
-// CIR: cir.store %{{[0-9]+}}, %[[OUT:[0-9]+]] : !rec_Wide, !cir.ptr<!rec_Wide>
-// CIR: %[[OUTCAST:[0-9]+]] = cir.cast bitcast %[[OUT]] : !cir.ptr<!rec_Wide> -> !cir.ptr<!s128i>
-// CIR: %[[WV:[0-9]+]] = cir.load %[[OUTCAST]] : !cir.ptr<!s128i>, !s128i
+// CIR: %[[W:[0-9]+]] = cir.alloca "w" align(16) init : !cir.ptr<!rec_Wide>
+// CIR: cir.copy %[[INCAST]] align(16) to %[[W]] align(16) : !cir.ptr<!rec_Wide>
+// CIR: %[[PV:[0-9]+]] = cir.load align(4) %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
+// CIR: %[[WCAST:[0-9]+]] = cir.cast bitcast %[[W]] : !cir.ptr<!rec_Wide> -> !cir.ptr<!s128i>
+// CIR: %[[WV:[0-9]+]] = cir.load align(16) %[[WCAST]] : !cir.ptr<!s128i>, !s128i
// CIR: cir.call @vf(%[[PV]], %[[WV]]) : (!u64i, !s128i) -> !s32i
// LLVM-LABEL: define dso_local i32 @call_wide(
// LLVM-SAME: i64 %[[P:[0-9a-zA-Z._]+]], i128 %[[W:[0-9a-zA-Z._]+]])
// LLVM-CIR: store i128 %[[W]], ptr %[[IN:[0-9a-zA-Z._]+]], align 16
-// LLVM-CIR: %{{[0-9a-zA-Z._]+}} = load %struct.Wide, ptr %[[IN]], align 16
-// LLVM-CIR: %[[PV:[0-9a-zA-Z._]+]] = load i64, ptr %{{[0-9a-zA-Z._]+}}, align 8
-// LLVM-CIR: store %struct.Wide %{{[0-9a-zA-Z._]+}}, ptr %[[OUT:[0-9a-zA-Z._]+]], align 16
-// LLVM-CIR-NEXT: %[[WV:[0-9a-zA-Z._]+]] = load i128, ptr %[[OUT]], align 16
+// LLVM-CIR: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[WSLOT:[0-9a-zA-Z._]+]], ptr align 16 %[[IN]], i64 16, i1 false)
+// LLVM-CIR-NEXT: %[[PV:[0-9a-zA-Z._]+]] = load i64, ptr %{{[0-9a-zA-Z._]+}}, align 4
+// LLVM-CIR-NEXT: %[[WV:[0-9a-zA-Z._]+]] = load i128, ptr %[[WSLOT]], align 16
// LLVM-OGCG: %[[DIVE:[0-9a-zA-Z._]+]] = getelementptr inbounds nuw %struct.Wide, ptr %[[WSLOT:[0-9a-zA-Z._]+]], i32 0, i32 0
// LLVM-OGCG-NEXT: store i128 %[[W]], ptr %[[DIVE]], align 16
@@ -182,17 +174,16 @@ int call_wide_char(Pair2 p, WideChar w) { return vf(p, w); }
// CIR: %[[W:[0-9]+]] = cir.alloca "w" align(16) init : !cir.ptr<!rec_WideChar>
// CIR-NOT: cir.alloca "byval"
// CIR: cir.copy %arg1 align(16) to %[[W]] align(16) : !cir.ptr<!rec_WideChar>
-// CIR: %[[PV:[0-9]+]] = cir.load %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
+// CIR: %[[PV:[0-9]+]] = cir.load align(4) %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
// CIR-NEXT: %{{[0-9]+}} = cir.call @vf(%[[PV]], %[[W]]) : (!u64i, !cir.ptr<!rec_WideChar> {llvm.align = 16 : i64, llvm.byval = !rec_WideChar, llvm.noundef}) -> !s32i
// LLVM-LABEL: define dso_local i32 @call_wide_char(
// LLVM-SAME: i64 %[[P:[0-9a-zA-Z._]+]], ptr noundef byval(%struct.WideChar) align 16 %[[W:[0-9a-zA-Z._]+]])
// LLVM-CIR: %[[WSLOT:[0-9a-zA-Z._]+]] = alloca %struct.WideChar, align 16
// LLVM-CIR: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[WSLOT]], ptr align 16 %[[W]], i64 32, i1 false)
-// LLVM-CIR: %[[PV:[0-9a-zA-Z._]+]] = load i64, ptr %{{[0-9a-zA-Z._]+}}, align 8
-// LLVM-CIR-NEXT: %{{[0-9a-zA-Z._]+}} = call i32 (i64, ...) @vf(i64 %[[PV]], ptr noundef byval(%struct.WideChar) align 16 %[[WSLOT]])
// LLVM-OGCG-NOT: memcpy
-// LLVM-OGCG: %[[PV:[0-9a-zA-Z._]+]] = load i64, ptr %{{[0-9a-zA-Z._]+}}, align 4
+// LLVM: %[[PV:[0-9a-zA-Z._]+]] = load i64, ptr %{{[0-9a-zA-Z._]+}}, align 4
+// LLVM-CIR-NEXT: %{{[0-9a-zA-Z._]+}} = call i32 (i64, ...) @vf(i64 %[[PV]], ptr noundef byval(%struct.WideChar) align 16 %[[WSLOT]])
// LLVM-OGCG-NEXT: %{{[0-9a-zA-Z._]+}} = call i32 (i64, ...) @vf(i64 %[[PV]], ptr noundef byval(%struct.WideChar) align 16 %[[W]])
// A _BitInt narrower than a register is extended at the ellipsis, same as a
@@ -202,27 +193,24 @@ int ell_bitint17(Pair2 p, _BitInt(17) b) { return vf(p, b); }
// CIR-LABEL: cir.func {{.*}}@ell_bitint17(%arg0: !u64i loc({{.+}}), %arg1: !cir.int<s, 17, bitint> {llvm.noundef, llvm.signext} loc({{.+}})) -> !s32i
// CIR: cir.store %arg1, %[[BSLOT:[0-9]+]] : !cir.int<s, 17, bitint>, !cir.ptr<!cir.int<s, 17, bitint>>
// CIR: %[[BV:[0-9]+]] = cir.load align(4) %[[BSLOT]] : !cir.ptr<!cir.int<s, 17, bitint>>, !cir.int<s, 17, bitint>
-// CIR: %[[PV:[0-9]+]] = cir.load %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
+// CIR: %[[PV:[0-9]+]] = cir.load align(4) %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
// CIR: cir.call @vf(%[[PV]], %[[BV]]) : (!u64i, !cir.int<s, 17, bitint> {llvm.noundef, llvm.signext}) -> !s32i
-// LLVM-CIR-LABEL: define dso_local i32 @ell_bitint17(
-// LLVM-CIR-SAME: i64 %[[P:[0-9a-zA-Z._]+]], i17 noundef signext %[[B:[0-9a-zA-Z._]+]])
-// LLVM-OGCG-LABEL: define dso_local i32 @ell_bitint17(
-// LLVM-OGCG-SAME: i64 %[[P:[0-9a-zA-Z._]+]], i17 noundef signext %[[B:[0-9a-zA-Z._]+]])
+// LLVM-LABEL: define dso_local i32 @ell_bitint17(
+// LLVM-SAME: i64 %[[P:[0-9a-zA-Z._]+]], i17 noundef signext %[[B:[0-9a-zA-Z._]+]])
// LLVM: %[[EXT:[0-9a-zA-Z._]+]] = sext i17 %[[B]] to i32
// LLVM: store i32 %[[EXT]], ptr %[[BSLOT:[0-9a-zA-Z._]+]], align 4
// LLVM: %[[RE:[0-9a-zA-Z._]+]] = load i32, ptr %[[BSLOT]], align 4
// LLVM: %[[TR:[0-9a-zA-Z._]+]] = trunc i32 %[[RE]] to i17
// LLVM: %[[PV:[0-9a-zA-Z._]+]] = load i64, ptr %{{[0-9a-zA-Z._]+}}, align
-// LLVM-CIR: call i32 (i64, ...) @vf(i64 %[[PV]], i17 noundef signext %[[TR]])
-// LLVM-OGCG: call i32 (i64, ...) @vf(i64 %[[PV]], i17 noundef signext %[[TR]])
+// LLVM: call i32 (i64, ...) @vf(i64 %[[PV]], i17 noundef signext %[[TR]])
// A width between 33 and 63 widens to one register.
int ell_bitint48(Pair2 p, _BitInt(48) b) { return vf(p, b); }
// CIR-LABEL: cir.func {{.*}}@ell_bitint48(%arg0: !u64i loc({{.+}}), %arg1: !u64i {llvm.noundef} loc({{.+}})) -> !s32i
// CIR: %[[BV:[0-9]+]] = cir.load align(8) %{{[0-9]+}} : !cir.ptr<!cir.int<s, 48, bitint>>, !cir.int<s, 48, bitint>
-// CIR: %[[PV:[0-9]+]] = cir.load %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
+// CIR: %[[PV:[0-9]+]] = cir.load align(4) %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
// CIR: %[[OUTCAST:[0-9]+]] = cir.cast bitcast %[[OUT:[0-9]+]] : !cir.ptr<!u64i> -> !cir.ptr<!cir.int<s, 48, bitint>>
// CIR-NEXT: cir.store %[[BV]], %[[OUTCAST]] : !cir.int<s, 48, bitint>, !cir.ptr<!cir.int<s, 48, bitint>>
// CIR-NEXT: %[[ARG:[0-9]+]] = cir.load %[[OUT]] : !cir.ptr<!u64i>, !u64i
@@ -250,7 +238,7 @@ int ell_bitint96(Pair2 p, _BitInt(96) b) { return vf(p, b); }
// CIR: cir.store %arg1, %[[IN0]] : !u64i, !cir.ptr<!u64i>
// CIR: %[[IN1:[0-9]+]] = cir.get_member %[[IN]][1] {name = ""} : !cir.ptr<!rec_anon_struct1> -> !cir.ptr<!u64i>
// CIR: cir.store %arg2, %[[IN1]] : !u64i, !cir.ptr<!u64i>
-// CIR: %[[PV:[0-9]+]] = cir.load %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
+// CIR: %[[PV:[0-9]+]] = cir.load align(4) %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
// CIR: %[[OUT0:[0-9]+]] = cir.get_member %[[OUT:[0-9]+]][0] {name = ""} : !cir.ptr<!rec_anon_struct1> -> !cir.ptr<!u64i>
// CIR: %[[B0:[0-9]+]] = cir.load %[[OUT0]] : !cir.ptr<!u64i>, !u64i
// CIR: %[[OUT1:[0-9]+]] = cir.get_member %[[OUT]][1] {name = ""} : !cir.ptr<!rec_anon_struct1> -> !cir.ptr<!u64i>
@@ -286,7 +274,7 @@ int ell_bitint128(Pair2 p, _BitInt(128) b) { return vf(p, b); }
// CIR-LABEL: cir.func {{.*}}@ell_bitint128(%arg0: !u64i loc({{.+}}), %arg1: !s128i_bitint {llvm.noundef} loc({{.+}})) -> !s32i
// CIR: cir.store %arg1, %[[BSLOT:[0-9]+]] : !s128i_bitint, !cir.ptr<!s128i_bitint>
// CIR: %[[BV:[0-9]+]] = cir.load align(8) %[[BSLOT]] : !cir.ptr<!s128i_bitint>, !s128i_bitint
-// CIR: %[[PV:[0-9]+]] = cir.load %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
+// CIR: %[[PV:[0-9]+]] = cir.load align(4) %{{[0-9]+}} : !cir.ptr<!u64i>, !u64i
// CIR: cir.call @vf(%[[PV]], %[[BV]]) : (!u64i, !s128i_bitint {llvm.noundef}) -> !s32i
// LLVM-LABEL: define dso_local i32 @ell_bitint128(
diff --git a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-x87-union.cpp b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-x87-union.cpp
new file mode 100644
index 00000000000000..4bf52ba95efbef
--- /dev/null
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-x87-union.cpp
@@ -0,0 +1,428 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefixes=LLVM,LLVMCIR --input-file=%t-cir.ll %s
+// RUN: FileCheck --check-prefix=NOMOVE --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefixes=LLVM,OGCG --input-file=%t.ll %s
+// RUN: FileCheck --check-prefix=NOMOVE --input-file=%t.ll %s
+
+struct View {
+ unsigned long size;
+ const char *data;
+};
+
+union Value {
+ long double ld;
+ View view;
+ int i;
+};
+
+union WideInt {
+ long double ld;
+ __int128 i128;
+};
+
+struct Arg {
+ Value value;
+ unsigned char type;
+};
+
+// CIR-DAG: !rec_anon_struct = !cir.struct<{data !u64i, data !u64i}>
+// CIR-DAG: !rec_Value = !cir.union<"Value" {data !cir.long_double<!cir.f80>, data !rec_View, data !s32i}>
+// CIR-DAG: !rec_WideInt = !cir.union<"WideInt" {data !cir.long_double<!cir.f80>, data !s128i}>
+// CIR-DAG: !rec_Arg = !cir.struct<"Arg" {data !rec_Value, data !u8i}>
+
+// LLVM-DAG: %struct.Arg = type { %union.Value, i8 }
+// LLVM-DAG: %union.Value = type { x86_fp80 }
+// LLVM-DAG: %union.WideInt = type { x86_fp80 }
+
+Arg ret_arg(const Arg *p) { return *p; }
+
+// CIR-LABEL: cir.func {{.*}} @_Z7ret_argPK3Arg(%arg0: !cir.ptr<!rec_Arg> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.noalias, llvm.sret = !rec_Arg, llvm.writable} loc({{[^)]+}}), %arg1: !cir.ptr<!rec_Arg> {llvm.noundef} loc({{[^)]+}}))
+// CIR: cir.copy %{{[^ ]+}} align(16) to %arg0 align(16) : !cir.ptr<!rec_Arg>
+
+// LLVM-LABEL: define dso_local void @_Z7ret_argPK3Arg(ptr dead_on_unwind noalias writable sret(%struct.Arg) align 16 %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 32, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@_Z7ret_argPK3Arg(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+unsigned char take_arg(Arg a) { return a.type; }
+
+// CIR-LABEL: cir.func {{.*}} @_Z8take_arg3Arg(%arg0: !cir.ptr<!rec_Arg> {llvm.align = 16 : i64, llvm.byval = !rec_Arg, llvm.noundef} loc({{[^)]+}})) -> (!u8i {llvm.noundef, llvm.zeroext})
+// CIR: cir.copy %arg0 align(16) to %{{[^ ]+}} align(16) : !cir.ptr<!rec_Arg>
+
+// LLVM-LABEL: define dso_local noundef zeroext i8 @_Z8take_arg3Arg(ptr noundef byval(%struct.Arg) align 16 %{{[^,)]+}})
+// LLVMCIR: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 32, i1 false)
+// OGCG-NOT: call void @llvm.memcpy
+
+// NOMOVE-LABEL: define {{.*}}@_Z8take_arg3Arg(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+unsigned char call_take_arg(const Arg *p) { return take_arg(*p); }
+
+// CIR-LABEL: cir.func {{.*}} @_Z13call_take_argPK3Arg(
+// CIR: cir.call @_Z8take_arg3Arg(%{{[^,)]+}}) : (!cir.ptr<!rec_Arg> {llvm.align = 16 : i64, llvm.byval = !rec_Arg, llvm.noundef}) -> (!u8i {llvm.noundef, llvm.zeroext})
+
+// LLVM-LABEL: define dso_local noundef zeroext i8 @_Z13call_take_argPK3Arg(ptr noundef %{{[^,)]+}})
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[TMP:[^,)]+]], ptr align 16 %{{[^,)]+}}, i64 32, i1 false)
+// LLVM: call noundef zeroext i8 @_Z8take_arg3Arg(ptr noundef byval(%struct.Arg) align 16 %[[TMP]])
+
+// NOMOVE-LABEL: define {{.*}}@_Z13call_take_argPK3Arg(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+unsigned char call_arg_ptr(unsigned char (*f)(Arg), const Arg *p) {
+ return f(*p);
+}
+
+// CIR-LABEL: cir.func {{.*}} @_Z12call_arg_ptrPFh3ArgEPKS_(
+// CIR: cir.call %{{[^(]+}}(%{{[^,)]+}}) : (!cir.ptr<!cir.func<(!cir.ptr<!rec_Arg>) -> !u8i>>, !cir.ptr<!rec_Arg> {llvm.align = 16 : i64, llvm.byval = !rec_Arg, llvm.noundef}) -> (!u8i {llvm.noundef, llvm.zeroext})
+
+// LLVM-LABEL: define dso_local noundef zeroext i8 @_Z12call_arg_ptrPFh3ArgEPKS_(ptr noundef %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[PTMP:[^,)]+]], ptr align 16 %{{[^,)]+}}, i64 32, i1 false)
+// LLVM: call noundef zeroext i8 %{{[^(]+}}(ptr noundef byval(%struct.Arg) align 16 %[[PTMP]])
+
+// NOMOVE-LABEL: define {{.*}}@_Z12call_arg_ptrPFh3ArgEPKS_(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+void variadic(int n, ...);
+
+void call_variadic(const Arg *a, const Value *v) { variadic(2, *a, *v); }
+
+// CIR-LABEL: cir.func {{.*}} @_Z13call_variadicPK3ArgPK5Value(
+// CIR: cir.call @_Z8variadiciz(%{{[^,)]+}}, %{{[^,)]+}}, %{{[^,)]+}}, %{{[^,)]+}}) : (!s32i {llvm.noundef}, !cir.ptr<!rec_Arg> {llvm.align = 16 : i64, llvm.byval = !rec_Arg, llvm.noundef}, !u64i, !u64i) -> ()
+
+// LLVM-LABEL: define dso_local void @_Z13call_variadicPK3ArgPK5Value(ptr noundef %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVM: call void (i32, ...) @_Z8variadiciz(i32 noundef 2, ptr noundef byval(%struct.Arg) align 16 %{{[^,)]+}}, i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@_Z13call_variadicPK3ArgPK5Value(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+void store_args(Arg *a, int i, const Arg *p) { a[i] = ret_arg(p); }
+
+// CIR-LABEL: cir.func {{.*}} @_Z10store_argsP3ArgiPKS_(
+// CIR: cir.call @_Z7ret_argPK3Arg(%{{[^,)]+}}, %{{[^,)]+}}) : (!cir.ptr<!rec_Arg> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.sret = !rec_Arg, llvm.writable}, !cir.ptr<!rec_Arg> {llvm.noundef}) -> ()
+
+// LLVM-LABEL: define dso_local void @_Z10store_argsP3ArgiPKS_(ptr noundef %{{[^,)]+}}, i32 noundef %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVM: call void @_Z7ret_argPK3Arg(ptr dead_on_unwind writable sret(%struct.Arg) align 16 %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@_Z10store_argsP3ArgiPKS_(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+Value ret_value(const Value *p) { return *p; }
+
+// CIR-LABEL: cir.func {{.*}} @_Z9ret_valuePK5Value(%arg0: !cir.ptr<!rec_Value> {llvm.noundef} loc({{[^)]+}})) -> !rec_anon_struct
+// CIR: %[[RETVAL:.+]] = cir.alloca "__retval" align(16) : !cir.ptr<!rec_Value>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[RETVAL]] align(16) : !cir.ptr<!rec_Value>
+// CIR: %[[RETVIEW:.+]] = cir.cast bitcast %[[RETVAL]] : !cir.ptr<!rec_Value> -> !cir.ptr<!rec_anon_struct>
+// CIR: %[[RET:.+]] = cir.load align(16) %[[RETVIEW]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
+// CIR: cir.return %[[RET]] : !rec_anon_struct
+
+// LLVM-LABEL: define dso_local { i64, i64 } @_Z9ret_valuePK5Value(ptr noundef %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@_Z9ret_valuePK5Value(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+const char *take_value(Value v) { return v.view.data; }
+
+// CIR-LABEL: cir.func {{.*}} @_Z10take_value5Value(%arg0: !u64i loc({{[^)]+}}), %arg1: !u64i loc({{[^)]+}})) -> (!cir.ptr<!s8i> {llvm.noundef})
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_anon_struct>
+// CIR: %[[LOP:.+]] = cir.get_member %[[SLOT]][0] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: cir.store %arg0, %[[LOP]] : !u64i, !cir.ptr<!u64i>
+// CIR: %[[HIP:.+]] = cir.get_member %[[SLOT]][1] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: cir.store %arg1, %[[HIP]] : !u64i, !cir.ptr<!u64i>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_Value>
+// CIR: %[[V:.+]] = cir.alloca "v" align(16) init : !cir.ptr<!rec_Value>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[V]] align(16) : !cir.ptr<!rec_Value>
+
+// LLVM-LABEL: define dso_local noundef ptr @_Z10take_value5Value(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@_Z10take_value5Value(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+Value pass_value(Value v) { return v; }
+
+// CIR-LABEL: cir.func {{.*}} @_Z10pass_value5Value(%arg0: !u64i loc({{[^)]+}}), %arg1: !u64i loc({{[^)]+}})) -> !rec_anon_struct
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_anon_struct>
+// CIR: %[[LOP:.+]] = cir.get_member %[[SLOT]][0] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: cir.store %arg0, %[[LOP]] : !u64i, !cir.ptr<!u64i>
+// CIR: %[[HIP:.+]] = cir.get_member %[[SLOT]][1] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: cir.store %arg1, %[[HIP]] : !u64i, !cir.ptr<!u64i>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_Value>
+// CIR: %[[V:.+]] = cir.alloca "v" align(16) init : !cir.ptr<!rec_Value>
+// CIR: %[[RETVAL:.+]] = cir.alloca "__retval" align(16) : !cir.ptr<!rec_Value>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[V]] align(16) : !cir.ptr<!rec_Value>
+// CIR: cir.copy %[[V]] align(16) to %[[RETVAL]] align(16) : !cir.ptr<!rec_Value>
+// CIR: %[[RETVIEW:.+]] = cir.cast bitcast %[[RETVAL]] : !cir.ptr<!rec_Value> -> !cir.ptr<!rec_anon_struct>
+// CIR: %[[RET:.+]] = cir.load align(16) %[[RETVIEW]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
+// CIR: cir.return %[[RET]] : !rec_anon_struct
+
+// LLVM-LABEL: define dso_local { i64, i64 } @_Z10pass_value5Value(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@_Z10pass_value5Value(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+const char *call_take_value(const Value *p) { return take_value(*p); }
+
+// CIR-LABEL: cir.func {{.*}} @_Z15call_take_valuePK5Value(
+// CIR: %[[TMP:.+]] = cir.alloca "agg.tmp0" align(16) : !cir.ptr<!rec_Value>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_Value>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[TMP]] : !cir.ptr<!rec_Value> -> !cir.ptr<!rec_anon_struct>
+// CIR: %[[LOP:.+]] = cir.get_member %[[VIEW]][0] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: %[[LO:.+]] = cir.load align(16) %[[LOP]] : !cir.ptr<!u64i>, !u64i
+// CIR: %[[HIP:.+]] = cir.get_member %[[VIEW]][1] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: %[[HI:.+]] = cir.load align(8) %[[HIP]] : !cir.ptr<!u64i>, !u64i
+// CIR: cir.call @_Z10take_value5Value(%[[LO]], %[[HI]]) : (!u64i, !u64i) -> (!cir.ptr<!s8i> {llvm.noundef})
+
+// LLVM-LABEL: define dso_local noundef ptr @_Z15call_take_valuePK5Value(ptr noundef %{{[^,)]+}})
+// LLVM: call noundef ptr @_Z10take_value5Value(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@_Z15call_take_valuePK5Value(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+const char *call_value_ptr(const char *(*f)(Value), const Value *p) {
+ return f(*p);
+}
+
+// CIR-LABEL: cir.func {{.*}} @_Z14call_value_ptrPFPKc5ValueEPKS1_(
+// CIR: %[[TMP:.+]] = cir.alloca "agg.tmp0" align(16) : !cir.ptr<!rec_Value>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_Value>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[TMP]] : !cir.ptr<!rec_Value> -> !cir.ptr<!rec_anon_struct>
+// CIR: %[[LOP:.+]] = cir.get_member %[[VIEW]][0] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: %[[LO:.+]] = cir.load align(16) %[[LOP]] : !cir.ptr<!u64i>, !u64i
+// CIR: %[[HIP:.+]] = cir.get_member %[[VIEW]][1] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: %[[HI:.+]] = cir.load align(8) %[[HIP]] : !cir.ptr<!u64i>, !u64i
+// CIR: cir.call %{{[^(]+}}(%[[LO]], %[[HI]]) : (!cir.ptr<!cir.func<(!u64i, !u64i) -> !cir.ptr<!s8i>>>, !u64i, !u64i) -> (!cir.ptr<!s8i> {llvm.noundef})
+
+// LLVM-LABEL: define dso_local noundef ptr @_Z14call_value_ptrPFPKc5ValueEPKS1_(ptr noundef %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVM: call noundef ptr %{{[^(]+}}(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@_Z14call_value_ptrPFPKc5ValueEPKS1_(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+Value call_ret_ptr(Value (*f)(const Value *), const Value *p) { return f(p); }
+
+// CIR-LABEL: cir.func {{.*}} @_Z12call_ret_ptrPF5ValuePKS_ES1_(
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_anon_struct>
+// CIR: %[[RETVAL:.+]] = cir.alloca "__retval" align(16) : !cir.ptr<!rec_Value>
+// CIR: %[[RES:.+]] = cir.call %{{[^(]+}}(%{{[^,)]+}}) : (!cir.ptr<!cir.func<(!cir.ptr<!rec_Value>) -> !rec_anon_struct>>, !cir.ptr<!rec_Value> {llvm.noundef}) -> !rec_anon_struct
+// CIR: cir.store %[[RES]], %[[SLOT]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_Value>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[RETVAL]] align(16) : !cir.ptr<!rec_Value>
+
+// LLVM-LABEL: define dso_local { i64, i64 } @_Z12call_ret_ptrPF5ValuePKS_ES1_(ptr noundef %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVM: call { i64, i64 } %{{[^(]+}}(ptr noundef %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@_Z12call_ret_ptrPF5ValuePKS_ES1_(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+void store_values(Value *a, int i, const Value *p) { a[i] = ret_value(p); }
+
+// CIR-LABEL: cir.func {{.*}} @_Z12store_valuesP5ValueiPKS_(
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_anon_struct>
+// CIR: %[[TMP:.+]] = cir.alloca "ref.tmp0" align(16) : !cir.ptr<!rec_Value>
+// CIR: %[[RES:.+]] = cir.call @_Z9ret_valuePK5Value(%{{[^,)]+}}) : (!cir.ptr<!rec_Value> {llvm.noundef}) -> !rec_anon_struct
+// CIR: cir.store %[[RES]], %[[SLOT]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_Value>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_Value>
+
+// LLVM-LABEL: define dso_local void @_Z12store_valuesP5ValueiPKS_(ptr noundef %{{[^,)]+}}, i32 noundef %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVM: call { i64, i64 } @_Z9ret_valuePK5Value(ptr noundef %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@_Z12store_valuesP5ValueiPKS_(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+WideInt ret_wide(const WideInt *p) { return *p; }
+
+// CIR-LABEL: cir.func {{.*}} @_Z8ret_widePK7WideInt(%arg0: !cir.ptr<!rec_WideInt> {llvm.noundef} loc({{[^)]+}})) -> !s128i
+// CIR: %[[RETVAL:.+]] = cir.alloca "__retval" align(16) : !cir.ptr<!rec_WideInt>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[RETVAL]] align(16) : !cir.ptr<!rec_WideInt>
+// CIR: %[[RETVIEW:.+]] = cir.cast bitcast %[[RETVAL]] : !cir.ptr<!rec_WideInt> -> !cir.ptr<!s128i>
+// CIR: %[[RET:.+]] = cir.load align(16) %[[RETVIEW]] : !cir.ptr<!s128i>, !s128i
+// CIR: cir.return %[[RET]] : !s128i
+
+// LLVMCIR-LABEL: define dso_local i128 @_Z8ret_widePK7WideInt(ptr noundef %{{[^,)]+}})
+// OGCG-LABEL: define dso_local { i64, i64 } @_Z8ret_widePK7WideInt(ptr noundef %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@_Z8ret_widePK7WideInt(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+__int128 take_wide(WideInt w) { return w.i128; }
+
+// CIR-LABEL: cir.func {{.*}} @_Z9take_wide7WideInt(%arg0: !s128i loc({{[^)]+}})) -> (!s128i {llvm.noundef})
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!s128i>
+// CIR: cir.store %arg0, %[[SLOT]] : !s128i, !cir.ptr<!s128i>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!s128i> -> !cir.ptr<!rec_WideInt>
+// CIR: %[[W:.+]] = cir.alloca "w" align(16) init : !cir.ptr<!rec_WideInt>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[W]] align(16) : !cir.ptr<!rec_WideInt>
+
+// LLVMCIR-LABEL: define dso_local noundef i128 @_Z9take_wide7WideInt(i128 %{{[^,)]+}})
+// OGCG-LABEL: define dso_local noundef i128 @_Z9take_wide7WideInt(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@_Z9take_wide7WideInt(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+__int128 call_take_wide(const WideInt *p) { return take_wide(*p); }
+
+// CIR-LABEL: cir.func {{.*}} @_Z14call_take_widePK7WideInt(
+// CIR: %[[TMP:.+]] = cir.alloca "agg.tmp0" align(16) : !cir.ptr<!rec_WideInt>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_WideInt>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[TMP]] : !cir.ptr<!rec_WideInt> -> !cir.ptr<!s128i>
+// CIR: %[[ARG:.+]] = cir.load align(16) %[[VIEW]] : !cir.ptr<!s128i>, !s128i
+// CIR: cir.call @_Z9take_wide7WideInt(%[[ARG]]) : (!s128i) -> (!s128i {llvm.noundef})
+
+// LLVM-LABEL: define dso_local noundef i128 @_Z14call_take_widePK7WideInt(ptr noundef %{{[^,)]+}})
+// LLVMCIR: call noundef i128 @_Z9take_wide7WideInt(i128 %{{[^,)]+}})
+// OGCG: call noundef i128 @_Z9take_wide7WideInt(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@_Z14call_take_widePK7WideInt(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+void store_wides(WideInt *a, int i, const WideInt *p) { a[i] = ret_wide(p); }
+
+// CIR-LABEL: cir.func {{.*}} @_Z11store_widesP7WideIntiPKS_(
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!s128i>
+// CIR: %[[TMP:.+]] = cir.alloca "ref.tmp0" align(16) : !cir.ptr<!rec_WideInt>
+// CIR: %[[RES:.+]] = cir.call @_Z8ret_widePK7WideInt(%{{[^,)]+}}) : (!cir.ptr<!rec_WideInt> {llvm.noundef}) -> !s128i
+// CIR: cir.store %[[RES]], %[[SLOT]] : !s128i, !cir.ptr<!s128i>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!s128i> -> !cir.ptr<!rec_WideInt>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_WideInt>
+
+// LLVM-LABEL: define dso_local void @_Z11store_widesP7WideIntiPKS_(ptr noundef %{{[^,)]+}}, i32 noundef %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
+// LLVMCIR: call i128 @_Z8ret_widePK7WideInt(ptr noundef %{{[^,)]+}})
+// OGCG: call { i64, i64 } @_Z8ret_widePK7WideInt(ptr noundef %{{[^,)]+}})
+
+// NOMOVE-LABEL: define {{.*}}@_Z11store_widesP7WideIntiPKS_(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+Value va_value(int n, ...) {
+ __builtin_va_list ap;
+ __builtin_va_start(ap, n);
+ Value v = __builtin_va_arg(ap, Value);
+ __builtin_va_end(ap);
+ return v;
+}
+
+// CIR-LABEL: cir.func {{.*}} @_Z8va_valueiz(
+// CIR: %[[TMP:.+]] = cir.alloca "vaarg.tmp" align(16) : !cir.ptr<!rec_Value>
+// CIR: %[[REG:.+]] = cir.alloca "vaarg.reg" align(16) : !cir.ptr<!rec_Value>
+// CIR: %[[ADDR:.+]] = cir.ternary(
+// CIR: %[[REGSLOT:.+]] = cir.cast bitcast %{{[^ ]+}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_Value>
+// CIR: cir.copy %[[REGSLOT]] align(8) to %[[REG]] align(16) : !cir.ptr<!rec_Value>
+// CIR: %[[RES:.+]] = cir.cast bitcast %[[ADDR]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_Value>
+// CIR: cir.copy %[[RES]] align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_Value>
+
+// LLVM-LABEL: define dso_local { i64, i64 } @_Z8va_valueiz(i32 noundef %{{[^,)]+}}, ...)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 8 %{{[^,)]+}}, i64 16, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@_Z8va_valueiz(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+WideInt va_wide(int n, ...) {
+ __builtin_va_list ap;
+ __builtin_va_start(ap, n);
+ WideInt w = __builtin_va_arg(ap, WideInt);
+ __builtin_va_end(ap);
+ return w;
+}
+
+// CIR-LABEL: cir.func {{.*}} @_Z7va_wideiz(
+// CIR: %[[TMP:.+]] = cir.alloca "vaarg.tmp" align(16) : !cir.ptr<!rec_WideInt>
+// CIR: %[[REG:.+]] = cir.alloca "vaarg.reg" align(16) : !cir.ptr<!rec_WideInt>
+// CIR: %[[ADDR:.+]] = cir.ternary(
+// CIR: %[[REGSLOT:.+]] = cir.cast bitcast %{{[^ ]+}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_WideInt>
+// CIR: cir.copy %[[REGSLOT]] align(8) to %[[REG]] align(16) : !cir.ptr<!rec_WideInt>
+// CIR: %[[RES:.+]] = cir.cast bitcast %[[ADDR]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_WideInt>
+// CIR: cir.copy %[[RES]] align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_WideInt>
+
+// LLVMCIR-LABEL: define dso_local i128 @_Z7va_wideiz(i32 noundef %{{[^,)]+}}, ...)
+// OGCG-LABEL: define dso_local { i64, i64 } @_Z7va_wideiz(i32 noundef %{{[^,)]+}}, ...)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 8 %{{[^,)]+}}, i64 16, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@_Z7va_wideiz(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+Arg va_arg_arg(int n, ...) {
+ __builtin_va_list ap;
+ __builtin_va_start(ap, n);
+ Arg a = __builtin_va_arg(ap, Arg);
+ __builtin_va_end(ap);
+ return a;
+}
+
+// CIR-LABEL: cir.func {{.*}} @_Z10va_arg_argiz(%arg0: !cir.ptr<!rec_Arg> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.noalias, llvm.sret = !rec_Arg, llvm.writable} loc({{[^)]+}}), %arg1: !s32i {llvm.noundef} loc({{[^)]+}}), ...)
+// CIR: %[[TMP:.+]] = cir.alloca "vaarg.tmp" align(16) : !cir.ptr<!rec_Arg>
+// CIR: %[[RES:.+]] = cir.cast bitcast %{{[^ ]+}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_Arg>
+// CIR: cir.copy %[[RES]] align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_Arg>
+// CIR: cir.copy %[[TMP]] align(16) to %arg0 align(16) : !cir.ptr<!rec_Arg>
+
+// LLVM-LABEL: define dso_local void @_Z10va_arg_argiz(ptr dead_on_unwind noalias writable sret(%struct.Arg) align 16 %{{[^,)]+}}, i32 noundef %{{[^,)]+}}, ...)
+// LLVMCIR: %[[TMP:.+]] = alloca %struct.Arg, align 16
+// LLVMCIR: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[TMP]], ptr align 16 %{{[^,)]+}}, i64 32, i1 false)
+// LLVMCIR: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %[[TMP]], i64 32, i1 false)
+// OGCG: %[[AREA:.+]] = call ptr @llvm.ptrmask.p0.i64(
+// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %[[AREA]], i64 32, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@_Z10va_arg_argiz(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<[0-9]+ x x86_fp80>|<?\{ ?x86_fp80|%struct\.|%union\.)}}
+
+// A union of only long doubles classifies X87, so it is returned as an
+// x86_fp80 read out of the record's memory and passed in memory.
+union X87U { long double a, b; };
+X87U make_x87u();
+
+X87U ret_x87u(const X87U *p) { return *p; }
+
+// CIR-LABEL: cir.func {{.*}} @_Z8ret_x87uPK4X87U(%arg0: !cir.ptr<!rec_X87U> {llvm.noundef} loc({{[^)]+}})) -> !cir.f80
+// CIR: %[[RV:.+]] = cir.alloca "__retval" align(16) : !cir.ptr<!rec_X87U>
+// CIR: cir.copy %{{[^ ]+}} align(16) to %[[RV]] align(16) : !cir.ptr<!rec_X87U>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[RV]] : !cir.ptr<!rec_X87U> -> !cir.ptr<!cir.f80>
+// CIR: %[[LD:.+]] = cir.load align(16) %[[VIEW]] : !cir.ptr<!cir.f80>, !cir.f80
+// CIR: cir.return %[[LD]] : !cir.f80
+
+// LLVM-LABEL: define dso_local x86_fp80 @_Z8ret_x87uPK4X87U(ptr noundef %{{[^,)]+}})
+// LLVM: %[[RV:.+]] = alloca %union.X87U, align 16
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[RV]], ptr align 16 %{{[^,)]+}}, i64 16, i1 false)
+// LLVM: %[[LD:.+]] = load x86_fp80, ptr %[[RV]], align 16
+// LLVM: ret x86_fp80 %[[LD]]
+
+// NOMOVE-LABEL: define {{.*}}@_Z8ret_x87uPK4X87U(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* %union\.}}
+
+long double take_x87u(X87U u) { return u.b; }
+
+// CIR-LABEL: cir.func {{.*}} @_Z9take_x87u4X87U(%arg0: !cir.ptr<!rec_X87U> {llvm.align = 16 : i64, llvm.byval = !rec_X87U, llvm.noundef} loc({{[^)]+}})) -> (!cir.long_double<!cir.f80> {llvm.noundef})
+// CIR: %[[U:.+]] = cir.alloca "u" align(16) init : !cir.ptr<!rec_X87U>
+// CIR: cir.copy %arg0 align(16) to %[[U]] align(16) : !cir.ptr<!rec_X87U>
+
+// LLVM-LABEL: define dso_local noundef x86_fp80 @_Z9take_x87u4X87U(ptr noundef byval(%union.X87U) align 16 %{{[^,)]+}})
+// LLVMCIR: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 16, i1 false)
+// OGCG-NOT: call void @llvm.memcpy
+// LLVM: load x86_fp80, ptr %{{[^,)]+}}, align 16
+
+// NOMOVE-LABEL: define {{.*}}@_Z9take_x87u4X87U(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* %union\.}}
+
+void store_x87u(X87U *q) { *q = make_x87u(); }
+
+// CIR-LABEL: cir.func {{.*}} @_Z10store_x87uP4X87U(
+// CIR: %[[SLOT:.+]] = cir.alloca "coerce" align(16) : !cir.ptr<!cir.f80>
+// CIR: %[[TMP:.+]] = cir.alloca "ref.tmp0" align(16) : !cir.ptr<!rec_X87U>
+// CIR: %[[RES:.+]] = cir.call @_Z9make_x87uv() : () -> !cir.f80
+// CIR: cir.store %[[RES]], %[[SLOT]] : !cir.f80, !cir.ptr<!cir.f80>
+// CIR: %[[VIEW:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!cir.f80> -> !cir.ptr<!rec_X87U>
+// CIR: cir.copy %[[VIEW]] align(16) to %[[TMP]] align(16) : !cir.ptr<!rec_X87U>
+
+// LLVM-LABEL: define dso_local void @_Z10store_x87uP4X87U(ptr noundef %{{[^,)]+}})
+// LLVMCIR: %[[SLOT:.+]] = alloca x86_fp80, align 16
+// LLVM: %[[TMP:.+]] = alloca %union.X87U, align 16
+// LLVM: %[[RES:.+]] = call x86_fp80 @_Z9make_x87uv()
+// LLVMCIR: store x86_fp80 %[[RES]], ptr %[[SLOT]], align 16
+// LLVMCIR: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[TMP]], ptr align 16 %[[SLOT]], i64 16, i1 false)
+// LLVMCIR: call noundef nonnull align 16 dereferenceable(16) ptr @_ZN4X87UaSEOS_(ptr noundef nonnull align 16 dereferenceable(16) %{{[^,)]+}}, ptr noundef nonnull align 16 dereferenceable(16) %[[TMP]])
+// OGCG: store x86_fp80 %[[RES]], ptr %[[TMP]], align 16
+// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %[[TMP]], i64 16, i1 false)
+
+// NOMOVE-LABEL: define {{.*}}@_Z10store_x87uP4X87U(
+// NOMOVE-NOT: {{(load|store)( volatile| atomic)* %union\.}}
diff --git a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-x87-vector.c b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-x87-vector.c
index 128ce8c2014079..52f8f6a705a82e 100644
--- a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-x87-vector.c
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-x87-vector.c
@@ -315,9 +315,7 @@ void var_ld3(void) { var(1, src_ld3()); }
// LLVM-AVX: call void (i32, ...) @var(i32 noundef 1, ptr noundef byval(<3 x x86_fp80>) align 64 %{{[^,)]+}})
// LLVM-AVX512: call void (i32, ...) @var(i32 noundef 1, <3 x x86_fp80> noundef %{{[^,)]+}})
-// A long double stores its first 10 bytes, so it can share a union with
-// members whose data ends within them, counting neither a record's tail
-// padding nor an empty array.
+// A long double next to an integer member classifies MEMORY.
struct __attribute__((aligned(16))) C16 { char c; };
union LDPad { long double ld; struct C16 s; };
union LDPad take_ldpad(union LDPad u) { return u; }
@@ -325,6 +323,7 @@ union LDPad take_ldpad(union LDPad u) { return u; }
// CIR: cir.func {{.*}}@take_ldpad(%arg0: !cir.ptr<!rec_LDPad> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.noalias, llvm.sret = !rec_LDPad, llvm.writable} loc({{[^)]+}}), %arg1: !cir.ptr<!rec_LDPad> {llvm.align = 16 : i64, llvm.byval = !rec_LDPad, llvm.noundef} loc({{[^)]+}}))
// LLVM: define dso_local void @take_ldpad(ptr dead_on_unwind noalias writable sret(%union.LDPad) align 16 %{{[^,)]+}}, ptr noundef byval(%union.LDPad) align 16 %{{[^,)]+}})
+// A zero-length array leaves the union X87.
union LDZero { long double ld; int z[0]; };
union LDZero take_ldzero(union LDZero u) { return u; }
diff --git a/clang/test/CIR/CodeGen/call.c b/clang/test/CIR/CodeGen/call.c
index 9a9c0c75bd4b37..dfc58315a0cdc2 100644
--- a/clang/test/CIR/CodeGen/call.c
+++ b/clang/test/CIR/CodeGen/call.c
@@ -17,18 +17,14 @@ void f2(void) {
}
// CIR-LABEL: cir.func{{.*}} @f2(){{.*}} {
-// CIR: %[[COERCE:.+]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_S>
-// CIR: %[[S:.+]] = cir.load align(4) %{{.+}} : !cir.ptr<!rec_S>, !rec_S
-// CIR-NEXT: cir.store %[[S]], %[[COERCE]] : !rec_S, !cir.ptr<!rec_S>
-// CIR-NEXT: %[[CAST:.+]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!rec_S> -> !cir.ptr<!u64i>
-// CIR-NEXT: %[[ARG:.+]] = cir.load %[[CAST]] : !cir.ptr<!u64i>, !u64i
+// CIR: %[[S:.+]] = cir.alloca "s" align(4) : !cir.ptr<!rec_S>
+// CIR-NEXT: %[[CAST:.+]] = cir.cast bitcast %[[S]] : !cir.ptr<!rec_S> -> !cir.ptr<!u64i>
+// CIR-NEXT: %[[ARG:.+]] = cir.load align(4) %[[CAST]] : !cir.ptr<!u64i>, !u64i
// CIR-NEXT: cir.call @f1(%[[ARG]]) : (!u64i) -> ()
// LLVM-LABEL: define{{.*}} void @f2(){{.*}}
-// LLVM: %[[COERCE:.+]] = alloca %struct.S, align 8
-// LLVM: %[[S:.+]] = load %struct.S, ptr %{{.+}}, align 4
-// LLVM-NEXT: store %struct.S %[[S]], ptr %[[COERCE]], align 4
-// LLVM-NEXT: %[[ARG:.+]] = load i64, ptr %[[COERCE]], align 8
+// LLVM: %[[S:.+]] = alloca %struct.S, align 4
+// LLVM-NEXT: %[[ARG:.+]] = load i64, ptr %[[S]], align 4
// LLVM-NEXT: call void @f1(i64 %[[ARG]])
// OGCG-LABEL: define{{.*}} void @f2()
@@ -42,18 +38,18 @@ void f4(void) {
// CIR-LABEL: cir.func{{.*}} @f4(){{.*}} {
// CIR: %[[COERCE:.+]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
-// CIR: %[[RET:.+]] = cir.call @f3() : () -> !u64i
+// CIR-NEXT: %[[S:.+]] = cir.alloca "s" align(4) init : !cir.ptr<!rec_S>
+// CIR-NEXT: %[[RET:.+]] = cir.call @f3() : () -> !u64i
// CIR-NEXT: cir.store %[[RET]], %[[COERCE]] : !u64i, !cir.ptr<!u64i>
// CIR-NEXT: %[[CAST:.+]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_S>
-// CIR-NEXT: %[[S:.+]] = cir.load %[[CAST]] : !cir.ptr<!rec_S>, !rec_S
-// CIR-NEXT: cir.store align(4) %[[S]], %{{.+}} : !rec_S, !cir.ptr<!rec_S>
+// CIR-NEXT: cir.copy %[[CAST]] align(8) to %[[S]] align(4) : !cir.ptr<!rec_S>
// LLVM-LABEL: define{{.*}} void @f4(){{.*}} {
// LLVM: %[[COERCE:.+]] = alloca i64, align 8
-// LLVM: %[[RET:.+]] = call i64 @f3()
+// LLVM-NEXT: %[[S:.+]] = alloca %struct.S, align 4
+// LLVM-NEXT: %[[RET:.+]] = call i64 @f3()
// LLVM-NEXT: store i64 %[[RET]], ptr %[[COERCE]], align 8
-// LLVM-NEXT: %[[S:.+]] = load %struct.S, ptr %[[COERCE]], align 4
-// LLVM-NEXT: store %struct.S %[[S]], ptr %{{.+}}, align 4
+// LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[S]], ptr align 8 %[[COERCE]], i64 8, i1 false)
// OGCG-LABEL: define{{.*}} void @f4() #0 {
// OGCG: %[[S:.+]] = call i64 @f3()
@@ -109,30 +105,22 @@ void f9(void) {
// CIR-LABEL: cir.func{{.*}} @f9(){{.*}} {
// CIR: %[[RETSLOT:.+]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
-// CIR-NEXT: %[[ARGSLOT:.+]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_S>
// CIR-NEXT: %[[SLOT:.+]] = cir.alloca "agg.tmp0" align(4) : !cir.ptr<!rec_S>
// CIR-NEXT: %[[RET:.+]] = cir.call @f3() : () -> !u64i
// CIR-NEXT: cir.store %[[RET]], %[[RETSLOT]] : !u64i, !cir.ptr<!u64i>
// CIR-NEXT: %[[RETCAST:.+]] = cir.cast bitcast %[[RETSLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_S>
-// CIR-NEXT: %[[RETVAL:.+]] = cir.load %[[RETCAST]] : !cir.ptr<!rec_S>, !rec_S
-// CIR-NEXT: cir.store align(4) %[[RETVAL]], %[[SLOT]] : !rec_S, !cir.ptr<!rec_S>
-// CIR-NEXT: %[[ARG:.+]] = cir.load align(4) %[[SLOT]] : !cir.ptr<!rec_S>, !rec_S
-// CIR-NEXT: cir.store %[[ARG]], %[[ARGSLOT]] : !rec_S, !cir.ptr<!rec_S>
-// CIR-NEXT: %[[ARGCAST:.+]] = cir.cast bitcast %[[ARGSLOT]] : !cir.ptr<!rec_S> -> !cir.ptr<!u64i>
-// CIR-NEXT: %[[ARGVAL:.+]] = cir.load %[[ARGCAST]] : !cir.ptr<!u64i>, !u64i
+// CIR-NEXT: cir.copy %[[RETCAST]] align(8) to %[[SLOT]] align(4) : !cir.ptr<!rec_S>
+// CIR-NEXT: %[[ARGCAST:.+]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_S> -> !cir.ptr<!u64i>
+// CIR-NEXT: %[[ARGVAL:.+]] = cir.load align(4) %[[ARGCAST]] : !cir.ptr<!u64i>, !u64i
// CIR-NEXT: cir.call @f1(%[[ARGVAL]]) : (!u64i) -> ()
// LLVM-LABEL: define{{.*}} void @f9(){{.*}} {
// LLVM: %[[RETSLOT:.+]] = alloca i64, align 8
-// LLVM-NEXT: %[[ARGSLOT:.+]] = alloca %struct.S, align 8
// LLVM-NEXT: %[[SLOT:.+]] = alloca %struct.S, align 4
// LLVM-NEXT: %[[RET:.+]] = call i64 @f3()
// LLVM-NEXT: store i64 %[[RET]], ptr %[[RETSLOT]], align 8
-// LLVM-NEXT: %[[RETVAL:.+]] = load %struct.S, ptr %[[RETSLOT]], align 4
-// LLVM-NEXT: store %struct.S %[[RETVAL]], ptr %[[SLOT]], align 4
-// LLVM-NEXT: %[[ARG:.+]] = load %struct.S, ptr %[[SLOT]], align 4
-// LLVM-NEXT: store %struct.S %[[ARG]], ptr %[[ARGSLOT]], align 4
-// LLVM-NEXT: %[[ARGVAL:.+]] = load i64, ptr %[[ARGSLOT]], align 8
+// LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[SLOT]], ptr align 8 %[[RETSLOT]], i64 8, i1 false)
+// LLVM-NEXT: %[[ARGVAL:.+]] = load i64, ptr %[[SLOT]], align 4
// LLVM-NEXT: call void @f1(i64 %[[ARGVAL]])
// OGCG-LABEL: define{{.*}} void @f9() #0 {
diff --git a/clang/test/CIR/CodeGen/call.cpp b/clang/test/CIR/CodeGen/call.cpp
index b7ffce1ebc0a68..d80768579a8c9d 100644
--- a/clang/test/CIR/CodeGen/call.cpp
+++ b/clang/test/CIR/CodeGen/call.cpp
@@ -88,15 +88,13 @@ void f11() {
// CIR: %[[#ret:]] = cir.call @_Z3f10v() : () -> !u64i
// CIR-NEXT: cir.store %[[#ret]], %[[#coerce]] : !u64i, !cir.ptr<!u64i>
// CIR-NEXT: %[[#cast:]] = cir.cast bitcast %[[#coerce]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_S>
-// CIR-NEXT: %[[#s:]] = cir.load %[[#cast]] : !cir.ptr<!rec_S>, !rec_S
-// CIR-NEXT: cir.store align(4) %[[#s]], %{{.+}} : !rec_S, !cir.ptr<!rec_S>
+// CIR-NEXT: cir.copy %[[#cast]] align(8) to %{{.+}} align(4) : !cir.ptr<!rec_S>
// LLVM-LABEL: define{{.*}} void @_Z3f11v(){{.*}}
// LLVM: %[[#coerce:]] = alloca i64, align 8
// LLVM: %[[#ret:]] = call i64 @_Z3f10v()
// LLVM-NEXT: store i64 %[[#ret]], ptr %[[#coerce]], align 8
-// LLVM-NEXT: %[[#s:]] = load %struct.S, ptr %[[#coerce]], align 4
-// LLVM-NEXT: store %struct.S %[[#s]], ptr %{{.+}}, align 4
+// LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %{{.+}}, ptr align 8 %[[#coerce]], i64 8, i1 false)
// OGCG-LABEL: define{{.*}} void @_Z3f11v(){{.*}}
// OGCG: %[[S:.+]] = alloca %struct.S, align 4
@@ -113,16 +111,14 @@ void f12() {
// CIR-NEXT: %[[#ret:]] = cir.call @_Z3f10v() : () -> !u64i
// CIR-NEXT: cir.store %[[#ret]], %[[#coerce]] : !u64i, !cir.ptr<!u64i>
// CIR-NEXT: %[[#cast:]] = cir.cast bitcast %[[#coerce]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_S>
-// CIR-NEXT: %[[#val:]] = cir.load %[[#cast]] : !cir.ptr<!rec_S>, !rec_S
-// CIR-NEXT: cir.store align(4) %[[#val]], %[[#slot]] : !rec_S, !cir.ptr<!rec_S>
+// CIR-NEXT: cir.copy %[[#cast]] align(8) to %[[#slot]] align(4) : !cir.ptr<!rec_S>
// LLVM-LABEL: define{{.*}} void @_Z3f12v(){{.*}} {
// LLVM: %[[#coerce:]] = alloca i64, align 8
// LLVM-NEXT: %[[#slot:]] = alloca %struct.S, align 4
// LLVM-NEXT: %[[#ret:]] = call i64 @_Z3f10v()
// LLVM-NEXT: store i64 %[[#ret]], ptr %[[#coerce]], align 8
-// LLVM-NEXT: %[[#val:]] = load %struct.S, ptr %[[#coerce]], align 4
-// LLVM-NEXT: store %struct.S %[[#val]], ptr %[[#slot]], align 4
+// LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[#slot]], ptr align 8 %[[#coerce]], i64 8, i1 false)
// OGCG-LABEL: define{{.*}} void @_Z3f12v(){{.*}}
// OGCG: %[[COERCE:.+]] = alloca %struct.S, align 4
diff --git a/clang/test/CIR/CodeGen/cast.cpp b/clang/test/CIR/CodeGen/cast.cpp
index fdac3056f60d92..5a1f3614816e5e 100644
--- a/clang/test/CIR/CodeGen/cast.cpp
+++ b/clang/test/CIR/CodeGen/cast.cpp
@@ -175,11 +175,13 @@ int test_unser_defined_cast_with_aligned_load(B x) {
}
// CIR: cir.func{{.*}} @_Z41test_unser_defined_cast_with_aligned_load1B
-// CIR: cir.store %{{.*}}, %[[B_PTR:.*]] : !rec_B, !cir.ptr<!rec_B>
+// CIR: %[[B_PTR:.*]] = cir.alloca "x" align(8) init : !cir.ptr<!rec_B>
+// CIR: cir.copy %{{.*}} align(8) to %[[B_PTR]] align(8) : !cir.ptr<!rec_B>
// CIR: %[[CAST:.*]] = cir.call @_ZNK1BcvPiEv(%[[B_PTR]])
// CIR: %[[LOAD:.*]] = cir.load{{.*}} %[[CAST]]
// LLVM: define{{.*}} i32 @_Z41test_unser_defined_cast_with_aligned_load1B
-// LLVM: store %struct.B %{{.*}}, ptr %[[B_PTR:.*]], align 8
+// LLVM: %[[B_PTR:.*]] = alloca %struct.B, align 8
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[B_PTR]], ptr align 8 %{{.*}}, i64 8, i1 false)
// LLVM: %[[CAST:.*]] = call {{.*}} ptr @_ZNK1BcvPiEv(ptr {{.*}} %[[B_PTR]])
// LLVM: %[[LOAD:.*]] = load i32, ptr %[[CAST]]
diff --git a/clang/test/CIR/CodeGen/complex.cpp b/clang/test/CIR/CodeGen/complex.cpp
index 1a3c7e6b6d3af7..9e4d592ab6d899 100644
--- a/clang/test/CIR/CodeGen/complex.cpp
+++ b/clang/test/CIR/CodeGen/complex.cpp
@@ -456,29 +456,23 @@ bool foo19(double _Complex a, double _Complex b) {
// LLVM-LABEL: define dso_local noundef zeroext i1 @_Z5foo19CdS_(double %0, double %1, double %2, double %3)
-// LLVM: %[[B_ADDR:.*]] = alloca { double, double }, align 8
-// LLVM-NEXT: %[[B_SLOT:.*]] = alloca { double, double }, align 8
+// LLVM: %[[B_SLOT:.*]] = alloca { double, double }, align 8
// LLVM-NEXT: %[[B_SLOT_REALP:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[B_SLOT]], i32 0, i32 0
// LLVM-NEXT: store double %2, ptr %[[B_SLOT_REALP]], align 8
// LLVM-NEXT: %[[B_SLOT_IMAGP:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[B_SLOT]], i32 0, i32 1
// LLVM-NEXT: store double %3, ptr %[[B_SLOT_IMAGP]], align 8
-// LLVM-NEXT: %[[B_RELOAD1:.*]] = load { double, double }, ptr %[[B_SLOT]], align 8
-// LLVM-NEXT: store { double, double } %[[B_RELOAD1]], ptr %[[B_ADDR]], align 8
-// LLVM-NEXT: %[[B_RELOAD2:.*]] = load { double, double }, ptr %[[B_ADDR]], align 8
-// LLVM-NEXT: %[[A_ADDR:.*]] = alloca { double, double }, align 8
+// LLVM-NEXT: %[[B_RELOAD:.*]] = load { double, double }, ptr %[[B_SLOT]], align 8
// LLVM-NEXT: %[[A_SLOT:.*]] = alloca { double, double }, align 8
// LLVM-NEXT: %[[A_SLOT_REALP:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[A_SLOT]], i32 0, i32 0
// LLVM-NEXT: store double %0, ptr %[[A_SLOT_REALP]], align 8
// LLVM-NEXT: %[[A_SLOT_IMAGP:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[A_SLOT]], i32 0, i32 1
// LLVM-NEXT: store double %1, ptr %[[A_SLOT_IMAGP]], align 8
-// LLVM-NEXT: %[[A_RELOAD1:.*]] = load { double, double }, ptr %[[A_SLOT]], align 8
-// LLVM-NEXT: store { double, double } %[[A_RELOAD1]], ptr %[[A_ADDR]], align 8
-// LLVM-NEXT: %[[A_RELOAD2:.*]] = load { double, double }, ptr %[[A_ADDR]], align 8
+// LLVM-NEXT: %[[A_RELOAD:.*]] = load { double, double }, ptr %[[A_SLOT]], align 8
// LLVM-NEXT: %[[REAL_A_ADDR:.*]] = alloca { double, double }, align 8
// LLVM-NEXT: %[[REAL_B_ADDR:.*]] = alloca { double, double }, align 8
// LLVM-NEXT: alloca i8, align 1
-// LLVM-NEXT: store { double, double } %[[A_RELOAD2]], ptr %[[REAL_A_ADDR]], align 8
-// LLVM-NEXT: store { double, double } %[[B_RELOAD2]], ptr %[[REAL_B_ADDR]], align 8
+// LLVM-NEXT: store { double, double } %[[A_RELOAD]], ptr %[[REAL_A_ADDR]], align 8
+// LLVM-NEXT: store { double, double } %[[B_RELOAD]], ptr %[[REAL_B_ADDR]], align 8
// LLVM-NEXT: %[[COMPLEX_A:.*]] = load { double, double }, ptr %[[REAL_A_ADDR]], align 8
// LLVM-NEXT: %[[COMPLEX_B:.*]] = load { double, double }, ptr %[[REAL_B_ADDR]], align 8
// LLVM: %[[A_REAL:.*]] = extractvalue { double, double } %[[COMPLEX_A]], 0
@@ -569,29 +563,23 @@ bool foo21(double _Complex a, double _Complex b) {
// CIR: %[[RESULT:.*]] = cir.cmp ne %[[COMPLEX_A]], %[[COMPLEX_B]] : !cir.complex<!cir.double>
// LLVM-LABEL: define dso_local noundef zeroext i1 @_Z5foo21CdS_(double %0, double %1, double %2, double %3)
-// LLVM: %[[B_ADDR:.*]] = alloca { double, double }, align 8
-// LLVM-NEXT: %[[B_SLOT:.*]] = alloca { double, double }, align 8
+// LLVM: %[[B_SLOT:.*]] = alloca { double, double }, align 8
// LLVM-NEXT: %[[B_SLOT_REALP:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[B_SLOT]], i32 0, i32 0
// LLVM-NEXT: store double %2, ptr %[[B_SLOT_REALP]], align 8
// LLVM-NEXT: %[[B_SLOT_IMAGP:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[B_SLOT]], i32 0, i32 1
// LLVM-NEXT: store double %3, ptr %[[B_SLOT_IMAGP]], align 8
-// LLVM-NEXT: %[[B_RELOAD1:.*]] = load { double, double }, ptr %[[B_SLOT]], align 8
-// LLVM-NEXT: store { double, double } %[[B_RELOAD1]], ptr %[[B_ADDR]], align 8
-// LLVM-NEXT: %[[B_RELOAD2:.*]] = load { double, double }, ptr %[[B_ADDR]], align 8
-// LLVM-NEXT: %[[A_ADDR:.*]] = alloca { double, double }, align 8
+// LLVM-NEXT: %[[B_RELOAD:.*]] = load { double, double }, ptr %[[B_SLOT]], align 8
// LLVM-NEXT: %[[A_SLOT:.*]] = alloca { double, double }, align 8
// LLVM-NEXT: %[[A_SLOT_REALP:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[A_SLOT]], i32 0, i32 0
// LLVM-NEXT: store double %0, ptr %[[A_SLOT_REALP]], align 8
// LLVM-NEXT: %[[A_SLOT_IMAGP:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[A_SLOT]], i32 0, i32 1
// LLVM-NEXT: store double %1, ptr %[[A_SLOT_IMAGP]], align 8
-// LLVM-NEXT: %[[A_RELOAD1:.*]] = load { double, double }, ptr %[[A_SLOT]], align 8
-// LLVM-NEXT: store { double, double } %[[A_RELOAD1]], ptr %[[A_ADDR]], align 8
-// LLVM-NEXT: %[[A_RELOAD2:.*]] = load { double, double }, ptr %[[A_ADDR]], align 8
+// LLVM-NEXT: %[[A_RELOAD:.*]] = load { double, double }, ptr %[[A_SLOT]], align 8
// LLVM-NEXT: %[[REAL_A_ADDR:.*]] = alloca { double, double }, align 8
// LLVM-NEXT: %[[REAL_B_ADDR:.*]] = alloca { double, double }, align 8
// LLVM-NEXT: alloca i8, align 1
-// LLVM-NEXT: store { double, double } %[[A_RELOAD2]], ptr %[[REAL_A_ADDR]], align 8
-// LLVM-NEXT: store { double, double } %[[B_RELOAD2]], ptr %[[REAL_B_ADDR]], align 8
+// LLVM-NEXT: store { double, double } %[[A_RELOAD]], ptr %[[REAL_A_ADDR]], align 8
+// LLVM-NEXT: store { double, double } %[[B_RELOAD]], ptr %[[REAL_B_ADDR]], align 8
// LLVM-NEXT: %[[COMPLEX_A:.*]] = load { double, double }, ptr %[[REAL_A_ADDR]], align 8
// LLVM-NEXT: %[[COMPLEX_B:.*]] = load { double, double }, ptr %[[REAL_B_ADDR]], align 8
// LLVM: %[[A_REAL:.*]] = extractvalue { double, double } %[[COMPLEX_A]], 0
diff --git a/clang/test/CIR/CodeGen/cxx-rewritten-binary-operator.cpp b/clang/test/CIR/CodeGen/cxx-rewritten-binary-operator.cpp
index 94d7552e1081f1..5115b6fc540e8a 100644
--- a/clang/test/CIR/CodeGen/cxx-rewritten-binary-operator.cpp
+++ b/clang/test/CIR/CodeGen/cxx-rewritten-binary-operator.cpp
@@ -66,8 +66,6 @@ void cxx_rewritten_binary_operator_complex_expr() {
// CIR: %[[R_CPLX:.*]] = cir.load %[[R_PTR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
// CIR: cir.store {{.*}} %[[R_CPLX]], %[[R_ADDR]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
-// The difference between LLVM and OGCG is due to missing ABI lowering.
-
// LLVM: %[[A_ADDR:.*]] = alloca %struct.ComplexItem, align 1
// LLVM: %[[B_ADDR:.*]] = alloca %struct.ComplexItem, align 1
// LLVM: %[[R_ADDR:.*]] = alloca { i32, i32 }, align 4
@@ -124,10 +122,7 @@ void cxx_rewritten_binary_operator_aggr_expr() {
// CIR: %[[RESULT:.*]] = cir.call @_ZNK15SpaceshipResultltEi(%[[TMP_ADDR]], %[[CONST_0]]) : (!cir.ptr<!rec_SpaceshipResult> {llvm.align = 1 : i64, llvm.dereferenceable = 1 : i64, llvm.nonnull, llvm.noundef}, !s32i {llvm.noundef}) -> !s32i
// CIR: cir.store %[[RESULT]], %[[R_SLOT:.*]] : !s32i, !cir.ptr<!s32i>
// CIR: %[[R_PTR:.*]] = cir.cast bitcast %[[R_SLOT]] : !cir.ptr<!s32i> -> !cir.ptr<!rec_Result>
-// CIR: %[[R_VAL:.*]] = cir.load %[[R_PTR]] : !cir.ptr<!rec_Result>, !rec_Result
-// CIR: cir.store {{.*}} %[[R_VAL]], %[[R_ADDR]] : !rec_Result, !cir.ptr<!rec_Result>
-
-// The difference between LLVM and OGCG is due to missing ABI lowering.
+// CIR: cir.copy %[[R_PTR]] align(4) to %[[R_ADDR]] align(4) : !cir.ptr<!rec_Result>
// LLVM: %[[A_ADDR:.*]] = alloca %struct.Item, align 1
// LLVM: %[[B_ADDR:.*]] = alloca %struct.Item, align 1
@@ -137,8 +132,7 @@ void cxx_rewritten_binary_operator_aggr_expr() {
// LLVM: store %struct.SpaceshipResult poison, ptr %[[TMP_ADDR]], align 1
// LLVM: %[[RESULT:.*]] = call i32 @_ZNK15SpaceshipResultltEi(ptr noundef nonnull align 1 dereferenceable(1) %[[TMP_ADDR]], i32 noundef 0)
// LLVM: store i32 %[[RESULT]], ptr %[[R_SLOT:.*]], align 4
-// LLVM: %[[R_VAL:.*]] = load %struct.Result, ptr %[[R_SLOT]], align 4
-// LLVM: store %struct.Result %[[R_VAL]], ptr %[[R_ADDR]], align 4
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[R_ADDR]], ptr align 4 %[[R_SLOT]], i64 4, i1 false)
// OGCG: %[[A_ADDR:.*]] = alloca %struct.Item, align 1
// OGCG: %[[B_ADDR:.*]] = alloca %struct.Item, align 1
diff --git a/clang/test/CIR/CodeGen/global-decomp-decls.cpp b/clang/test/CIR/CodeGen/global-decomp-decls.cpp
index 65de2750ca2653..20e0704a445c2b 100644
--- a/clang/test/CIR/CodeGen/global-decomp-decls.cpp
+++ b/clang/test/CIR/CodeGen/global-decomp-decls.cpp
@@ -1,9 +1,9 @@
// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
-// RUN: FileCheck --check-prefix=LLVM,LLVMCIR --input-file=%t-cir.ll %s
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s
// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
-// RUN: FileCheck --check-prefix=LLVM,OGCG --input-file=%t.ll %s
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s
template<typename T>
auto getT() {
@@ -38,16 +38,13 @@ const auto & [t11, t12, t13] = getT<Type>();
// CIR: %[[GETTCALL:.*]] = cir.call @_Z4getTI4TypeEDav() : () -> !rec_anon_struct
// CIR: cir.store %[[GETTCALL]], %[[COERCE]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CIR: %[[COERCE_REC:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_Type>
-// CIR: %[[TVAL:.*]] = cir.load %[[COERCE_REC]] : !cir.ptr<!rec_Type>, !rec_Type
-// CIR: cir.store align(4) %[[TVAL]], %[[SB_REF]] : !rec_Type, !cir.ptr<!rec_Type>
+// CIR: cir.copy %[[COERCE_REC]] align(8) to %[[SB_REF]] align(4) : !cir.ptr<!rec_Type>
// CIR: cir.store align(8) %[[SB_REF]], %[[SB]] : !cir.ptr<!rec_Type>, !cir.ptr<!cir.ptr<!rec_Type>>
// LLVM: define internal void @__cxx_global_var_init{{.*}}()
// LLVM: %[[GETTCALL:.*]] = call { i64, i32 } @_Z4getTI4TypeEDav()
// LLVM: store { i64, i32 } %[[GETTCALL]], ptr %[[COERCED_PTR:.*]], align 8
-// LLVMCIR: %[[TVAL:.*]] = load %struct.Type, ptr %[[COERCED_PTR]], align 4
-// LLVMCIR: store %struct.Type %[[TVAL]], ptr @_ZGRDC3t113t123t13E_, align 4
-// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 4 @_ZGRDC3t113t123t13E_, ptr align 8 %[[COERCED_PTR]], i64 12, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 @_ZGRDC3t113t123t13E_, ptr align 8 %[[COERCED_PTR]], i64 12, i1 false)
// LLVM: store ptr @_ZGRDC3t113t123t13E_, ptr @_ZDC3t113t123t13E, align 8
struct DtorType { int a, b, c; ~DtorType(); };
diff --git a/clang/test/CIR/CodeGen/instantiate-init.cpp b/clang/test/CIR/CodeGen/instantiate-init.cpp
index 1dedf1f6f3fa70..42b82392925a26 100644
--- a/clang/test/CIR/CodeGen/instantiate-init.cpp
+++ b/clang/test/CIR/CodeGen/instantiate-init.cpp
@@ -24,7 +24,6 @@ void init_vec_using_initalizer_list() {
Vector vec = {0, 1, 2};
}
-// CIR: %[[COERCE:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!rec_std3A3Ainitializer_list3Cint3E>
// CIR: %[[VEC_ADDR:.*]] = cir.alloca "vec" {{.*}} init : !cir.ptr<!rec_Vector>
// CIR: %[[AGG_ADDR:.*]] = cir.alloca "agg.tmp0" {{.*}} : !cir.ptr<!rec_std3A3Ainitializer_list3Cint3E>
// CIR: %[[INIT_LIST_ADDR:.*]] = cir.alloca "ref.tmp0" {{.*}} : !cir.ptr<!cir.array<!s32i x 3>>
@@ -45,13 +44,11 @@ void init_vec_using_initalizer_list() {
// CIR: %[[CONST_U64_3:.*]] = cir.const #cir.int<3> : !u64i
// CIR: %[[SIZE_PTR:.*]] = cir.get_member %[[AGG_ADDR]][1] {name = "size"} : !cir.ptr<!rec_std3A3Ainitializer_list3Cint3E> -> !cir.ptr<!u64i>
// CIR: cir.store {{.*}} %[[CONST_U64_3]], %[[SIZE_PTR]] : !u64i, !cir.ptr<!u64i>
-// CIR: %[[TMP_AGG:.*]] = cir.load {{.*}} %[[AGG_ADDR]] : !cir.ptr<!rec_std3A3Ainitializer_list3Cint3E>, !rec_std3A3Ainitializer_list3Cint3E
-// CIR: cir.store %[[TMP_AGG]], %[[COERCE]] : !rec_std3A3Ainitializer_list3Cint3E, !cir.ptr<!rec_std3A3Ainitializer_list3Cint3E>
-// CIR: %[[COERCE_REC:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!rec_std3A3Ainitializer_list3Cint3E> -> !cir.ptr<!rec_anon_struct>
-// CIR: %[[DATA_SLOT:.*]] = cir.get_member %[[COERCE_REC]][0] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!cir.ptr<!void>>
-// CIR: %[[DATA_ARG:.*]] = cir.load %[[DATA_SLOT]] : !cir.ptr<!cir.ptr<!void>>, !cir.ptr<!void>
-// CIR: %[[SIZE_SLOT:.*]] = cir.get_member %[[COERCE_REC]][1] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
-// CIR: %[[SIZE_ARG:.*]] = cir.load %[[SIZE_SLOT]] : !cir.ptr<!u64i>, !u64i
+// CIR: %[[AGG_REC:.*]] = cir.cast bitcast %[[AGG_ADDR]] : !cir.ptr<!rec_std3A3Ainitializer_list3Cint3E> -> !cir.ptr<!rec_anon_struct>
+// CIR: %[[DATA_SLOT:.*]] = cir.get_member %[[AGG_REC]][0] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!cir.ptr<!void>>
+// CIR: %[[DATA_ARG:.*]] = cir.load align(8) %[[DATA_SLOT]] : !cir.ptr<!cir.ptr<!void>>, !cir.ptr<!void>
+// CIR: %[[SIZE_SLOT:.*]] = cir.get_member %[[AGG_REC]][1] {name = ""} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CIR: %[[SIZE_ARG:.*]] = cir.load align(8) %[[SIZE_SLOT]] : !cir.ptr<!u64i>, !u64i
// CIR: cir.call @_ZN6VectorC1ESt16initializer_listIiE(%[[VEC_ADDR]], %[[DATA_ARG]], %[[SIZE_ARG]]) : (!cir.ptr<!rec_Vector> {llvm.align = 1 : i64, llvm.dereferenceable = 1 : i64, llvm.nonnull, llvm.noundef}, !cir.ptr<!void>, !u64i) -> ()
// CIR: cir.cleanup.scope {
// CIR: cir.yield
@@ -60,7 +57,6 @@ void init_vec_using_initalizer_list() {
// CIR: cir.yield
// CIR: }
-// LLVM: %[[COERCE:.*]] = alloca %"class.std::initializer_list<int>", align 8
// LLVM: %[[VEC_ADDR:.*]] = alloca %struct.Vector, align 1
// LLVM: %[[AGG_ADDR:.*]] = alloca %"class.std::initializer_list<int>", align 8
// LLVM: %[[INIT_LIST_ADDR:.*]] = alloca [3 x i32], align 4
@@ -74,11 +70,9 @@ void init_vec_using_initalizer_list() {
// LLVM: store ptr %[[INIT_LIST_ADDR]], ptr %[[DATA_PTR]], align 8
// LLVM: %[[SIZE_PTR:.*]] = getelementptr inbounds nuw %"class.std::initializer_list<int>", ptr %[[AGG_ADDR]], i32 0, i32 1
// LLVM: store i64 3, ptr %[[SIZE_PTR]], align 8
-// LLVM: %[[TMP_AGG:.*]] = load %"class.std::initializer_list<int>", ptr %[[AGG_ADDR]], align 8
-// LLVM: store %"class.std::initializer_list<int>" %[[TMP_AGG]], ptr %[[COERCE]], align 8
-// LLVM: %[[DATA_SLOT:.*]] = getelementptr inbounds nuw { ptr, i64 }, ptr %[[COERCE]], i32 0, i32 0
+// LLVM: %[[DATA_SLOT:.*]] = getelementptr inbounds nuw { ptr, i64 }, ptr %[[AGG_ADDR]], i32 0, i32 0
// LLVM: %[[DATA_ARG:.*]] = load ptr, ptr %[[DATA_SLOT]], align 8
-// LLVM: %[[SIZE_SLOT:.*]] = getelementptr inbounds nuw { ptr, i64 }, ptr %[[COERCE]], i32 0, i32 1
+// LLVM: %[[SIZE_SLOT:.*]] = getelementptr inbounds nuw { ptr, i64 }, ptr %[[AGG_ADDR]], i32 0, i32 1
// LLVM: %[[SIZE_ARG:.*]] = load i64, ptr %[[SIZE_SLOT]], align 8
// LLVM: call void @_ZN6VectorC1ESt16initializer_listIiE(ptr noundef nonnull align 1 dereferenceable(1) %[[VEC_ADDR]], ptr %[[DATA_ARG]], i64 %[[SIZE_ARG]])
// LLVM: br label %[[SCOPE_CONT:.*]]
diff --git a/clang/test/CIR/CodeGen/lambda-explicit-obj.cpp b/clang/test/CIR/CodeGen/lambda-explicit-obj.cpp
index 18fd8821b20136..10c3a0bcb114c3 100644
--- a/clang/test/CIR/CodeGen/lambda-explicit-obj.cpp
+++ b/clang/test/CIR/CodeGen/lambda-explicit-obj.cpp
@@ -27,7 +27,6 @@ void g(HasThis ht) { ht.func(); }
// 'func' body.
// CIR-LABEL: cir.func {{.*}}@_ZN7HasThis4funcEv(
// CIR-SAME: %[[THIS:.*]]: !cir.ptr<!rec_HasThis>
-// CIR: %[[COERCE:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_anon2E0>
// CIR: %[[THIS_ALLOCA:.*]] = cir.alloca "this" align(8) init : !cir.ptr<!cir.ptr<!rec_HasThis>>
// CIR: %[[CAPTURE_ALLOCA:.*]] = cir.alloca "capturesThis" align(8) init : !cir.ptr<!rec_anon2E0>
// CIR: %[[TMP_ALLOCA:.*]] = cir.alloca "agg.tmp0" align(8) : !cir.ptr<!rec_anon2E0>
@@ -36,17 +35,13 @@ void g(HasThis ht) { ht.func(); }
// CIR: %[[CAPTURE_THIS_GET_MEM:.*]] = cir.get_member %[[CAPTURE_ALLOCA]][0] {name = "this"} : !cir.ptr<!rec_anon2E0> -> !cir.ptr<!cir.ptr<!rec_HasThis>>
// CIR: cir.store align(8) %[[THIS_LOAD]], %[[CAPTURE_THIS_GET_MEM]] : !cir.ptr<!rec_HasThis>, !cir.ptr<!cir.ptr<!rec_HasThis>>
// CIR: cir.copy %[[CAPTURE_ALLOCA]]{{.*}} to %[[TMP_ALLOCA]]{{.*}} : !cir.ptr<!rec_anon2E0>
-// CIR: %[[LOAD_TMP:.*]] = cir.load align(8) %[[TMP_ALLOCA]] : !cir.ptr<!rec_anon2E0>, !rec_anon2E0
-// CIR: cir.store %[[LOAD_TMP]], %[[COERCE]] : !rec_anon2E0, !cir.ptr<!rec_anon2E0>
-// CIR: %[[CAST_COERCE:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!rec_anon2E0> -> !cir.ptr<!cir.ptr<!void>>
-// CIR: %[[LOAD_COERCE:.*]] = cir.load %[[CAST_COERCE]] : !cir.ptr<!cir.ptr<!void>>, !cir.ptr<!void>
+// CIR: %[[CAST_TMP:.*]] = cir.cast bitcast %[[TMP_ALLOCA]] : !cir.ptr<!rec_anon2E0> -> !cir.ptr<!cir.ptr<!void>>
+// CIR: %[[LOAD_COERCE:.*]] = cir.load align(8) %[[CAST_TMP]] : !cir.ptr<!cir.ptr<!void>>, !cir.ptr<!void>
// CIR: cir.call @_ZZN7HasThis4funcEvENHUlT_E_clIS1_EEDaS0_(%[[LOAD_COERCE]]) : (!cir.ptr<!void>) -> ()
// LLVM-LABEL: define {{.*}}@_ZN7HasThis4funcEv
// LLVM-SAME: (ptr {{.*}}%[[THIS:.*]])
-// OGCG: %[[THIS_ALLOCA:.*]] = alloca ptr, align 8
-// CIRONLY: %[[COERCE:.*]] = alloca %[[LAMBDA_TY]], align 8
-// CIRONLY: %[[THIS_ALLOCA:.*]] = alloca ptr, align 8
+// LLVM: %[[THIS_ALLOCA:.*]] = alloca ptr, align 8
// LLVM: %[[CAPTURE_ALLOCA:.*]] = alloca %[[LAMBDA_TY]], align 8
// LLVM: %[[TMP_ALLOCA:.*]] = alloca %[[LAMBDA_TY]], align 8
// LLVM: store ptr %[[THIS]], ptr %[[THIS_ALLOCA]], align 8
@@ -54,10 +49,9 @@ void g(HasThis ht) { ht.func(); }
// LLVM: %[[CAPTURE_THIS_GET_MEM:.*]] = getelementptr inbounds nuw %[[LAMBDA_TY]], ptr %[[CAPTURE_ALLOCA]], i32 0, i32 0
// LLVM: store ptr %[[THIS_LOAD]], ptr %[[CAPTURE_THIS_GET_MEM]], align 8
// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr {{.*}}%[[TMP_ALLOCA]], ptr {{.*}}%[[CAPTURE_ALLOCA]], i64 8, i1 false)
-// CIRONLY: %[[LOAD_TMP:.*]] = load %[[LAMBDA_TY]], ptr %[[TMP:.*]], align 8
-// CIRONLY: store %[[LAMBDA_TY]] %[[LOAD_TMP]], ptr %[[COERCE]], align 8
+// CIRONLY: %[[LOAD_COERCE:.*]] = load ptr, ptr %[[TMP_ALLOCA]], align 8
// OGCG: %[[COERCE:.*]] = getelementptr inbounds nuw %[[LAMBDA_TY]], ptr %[[TMP_ALLOCA]], i32 0, i32 0
-// LLVM: %[[LOAD_COERCE:.*]] = load ptr, ptr %[[COERCE]], align 8
+// OGCG: %[[LOAD_COERCE:.*]] = load ptr, ptr %[[COERCE]], align 8
// LLVM: call void @_ZZN7HasThis4funcEvENHUlT_E_clIS1_EEDaS0_(ptr %[[LOAD_COERCE]])
// Lambda Body:
@@ -66,9 +60,8 @@ void g(HasThis ht) { ht.func(); }
// CIR: %[[COERCE:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!cir.ptr<!void>>
// CIR: cir.store %[[LAMBDA_THIS]], %[[COERCE]] : !cir.ptr<!void>, !cir.ptr<!cir.ptr<!void>>
// CIR: %[[CAST_COERCE:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!cir.ptr<!void>> -> !cir.ptr<!rec_anon2E0>
-// CIR: %[[LOAD_COERCE:.*]] = cir.load %[[CAST_COERCE]] : !cir.ptr<!rec_anon2E0>, !rec_anon2E0
// CIR: %[[THIS_ALLOCA:.*]] = cir.alloca "" align(8) init : !cir.ptr<!rec_anon2E0>
-// CIR: cir.store %[[LOAD_COERCE]], %[[THIS_ALLOCA]] : !rec_anon2E0, !cir.ptr<!rec_anon2E0>
+// CIR: cir.copy %[[CAST_COERCE]] align(8) to %[[THIS_ALLOCA]] align(8) : !cir.ptr<!rec_anon2E0>
// CIR: %[[GET_HAS_THIS:.*]] = cir.get_member %[[THIS_ALLOCA]][0] {name = "this"} : !cir.ptr<!rec_anon2E0> -> !cir.ptr<!cir.ptr<!rec_HasThis>>
// CIR: %[[LOAD_HAS_THIS:.*]] = cir.load align(8) %[[GET_HAS_THIS]] : !cir.ptr<!cir.ptr<!rec_HasThis>>, !cir.ptr<!rec_HasThis>
// CIR: cir.get_member %[[LOAD_HAS_THIS]][0] {name = "m"} : !cir.ptr<!rec_HasThis> -> !cir.ptr<!s32i>
@@ -81,9 +74,8 @@ void g(HasThis ht) { ht.func(); }
// LLVM: store ptr %[[LAMBDA_THIS]], ptr %[[COERCE]], align 8
-// CIRONLY: %[[LOAD_COERCE:.*]] = load %[[LAMBDA_TY]], ptr %[[COERCE]], align 8
// CIRONLY: %[[THIS_ALLOCA:.*]] = alloca %[[LAMBDA_TY]], align 8
-// CIRONLY: store %[[LAMBDA_TY]] %[[LOAD_COERCE]], ptr %[[THIS_ALLOCA]], align 8
+// CIRONLY: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[THIS_ALLOCA]], ptr align 8 %[[COERCE]], i64 8, i1 false)
// CIRONLY: %[[GET_HAS_THIS:.*]] = getelementptr inbounds nuw %[[LAMBDA_TY]], ptr %[[THIS_ALLOCA]], i32 0, i32 0
// OGCG: %[[GET_HAS_THIS:.*]] = getelementptr inbounds nuw %[[LAMBDA_TY]], ptr %[[LAMBDA_ALLOCA]], i32 0, i32 0
diff --git a/clang/test/CIR/CodeGen/lambda.cpp b/clang/test/CIR/CodeGen/lambda.cpp
index cf5d2be4dbb24d..44d83e0da8cf03 100644
--- a/clang/test/CIR/CodeGen/lambda.cpp
+++ b/clang/test/CIR/CodeGen/lambda.cpp
@@ -238,8 +238,7 @@ int f() {
// CIR: %[[G2:.*]] = cir.call @_Z2g2v() : () -> !cir.ptr<!void>
// CIR: cir.store %[[G2]], %[[COERCE]] : !cir.ptr<!void>, !cir.ptr<!cir.ptr<!void>>
// CIR: %[[COERCE_REC:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!cir.ptr<!void>> -> !cir.ptr<![[REC_LAM_G2]]>
-// CIR: %[[LAM:.*]] = cir.load %[[COERCE_REC]] : !cir.ptr<![[REC_LAM_G2]]>, ![[REC_LAM_G2]]
-// CIR: cir.store{{.*}} %[[LAM]], %[[TMP]]
+// CIR: cir.copy %[[COERCE_REC]] align(8) to %[[TMP]] align(8) : !cir.ptr<![[REC_LAM_G2]]>
// CIR: %[[RESULT:.*]] = cir.call @_ZZ2g2vENK3$_0clEv(%[[TMP]])
// CIR: cir.store{{.*}} %[[RESULT]], %[[RETVAL]]
// CIR: %[[RET:.*]] = cir.load{{.*}} %[[RETVAL]]
@@ -269,8 +268,7 @@ int f() {
// LLVM: %[[TMP:.*]] = alloca %[[REC_LAM_G2:[^,]*]],
// LLVM: %[[G2:.*]] = call ptr @_Z2g2v()
// LLVM: store ptr %[[G2]], ptr %[[COERCE]]
-// LLVM: %[[LAM:.*]] = load %[[REC_LAM_G2]], ptr %[[COERCE]]
-// LLVM: store %[[REC_LAM_G2]] %[[LAM]], ptr %[[TMP]]
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[TMP]], ptr align 8 %[[COERCE]], i64 8, i1 false)
// LLVM: %[[RESULT:.*]] = call {{.*}}i32 @"_ZZ2g2vENK3$_0clEv"(ptr {{.*}} %[[TMP]])
// LLVM: store i32 %[[RESULT]], ptr %[[RETVAL]]
// LLVM: %[[RET:.*]] = load i32, ptr %[[RETVAL]]
diff --git a/clang/test/CIR/CodeGen/non-scalar-lval-return.cpp b/clang/test/CIR/CodeGen/non-scalar-lval-return.cpp
index 2658bd0aee4d97..5525a6b7fd52a3 100644
--- a/clang/test/CIR/CodeGen/non-scalar-lval-return.cpp
+++ b/clang/test/CIR/CodeGen/non-scalar-lval-return.cpp
@@ -26,8 +26,7 @@ extern "C" void use() {
// CIR: %[[GET_STRUCT_CALL:.*]] = cir.call @getStruct(%[[ZERO]])
// CIR: cir.store{{.*}} %[[GET_STRUCT_CALL]], %[[COERCE]] : !s32i, !cir.ptr<!s32i>
// CIR: %[[COERCE_REC:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!s32i> -> !cir.ptr<!rec_Struct>
- // CIR: %[[STRUCT_VAL:.*]] = cir.load %[[COERCE_REC]] : !cir.ptr<!rec_Struct>, !rec_Struct
- // CIR: cir.store{{.*}} %[[STRUCT_VAL]], %[[TEMP_ALLOCA]]
+ // CIR: cir.copy %[[COERCE_REC]] align(4) to %[[TEMP_ALLOCA]] align(4) : !cir.ptr<!rec_Struct>
// CIR: %[[GET_MEMBER:.*]] = cir.get_member %[[TEMP_ALLOCA]][0] {name = "member"}
// CIR: %[[LOAD_MEM:.*]] = cir.load{{.*}}%[[GET_MEMBER]]
// CIR: cir.store{{.*}} %[[LOAD_MEM]], %[[G_ALLOCA]] : !s32i, !cir.ptr<!s32i>
@@ -38,8 +37,7 @@ extern "C" void use() {
// LLVMCIR: %[[TEMP_ALLOCA:.*]] = alloca %struct.Struct
// LLVMCIR: %[[GET_STRUCT_CALL:.*]] = call i32 @getStruct(i32 noundef 0)
// LLVMCIR: store i32 %[[GET_STRUCT_CALL]], ptr %[[COERCE]]
- // LLVMCIR: %[[STRUCT_VAL:.*]] = load %struct.Struct, ptr %[[COERCE]]
- // LLVMCIR: store %struct.Struct %[[STRUCT_VAL]], ptr %[[TEMP_ALLOCA]]
+ // LLVMCIR: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[TEMP_ALLOCA]], ptr align 4 %[[COERCE]], i64 4, i1 false)
//
// OGCG: %[[G_ALLOCA:.*]] = alloca i32
// OGCG: %[[TEMP_ALLOCA:.*]] = alloca %struct.Struct
diff --git a/clang/test/CIR/CodeGen/pointer-to-member-func-cast.cpp b/clang/test/CIR/CodeGen/pointer-to-member-func-cast.cpp
index 850b9ef9a918ce..82c47d9d5699b1 100644
--- a/clang/test/CIR/CodeGen/pointer-to-member-func-cast.cpp
+++ b/clang/test/CIR/CodeGen/pointer-to-member-func-cast.cpp
@@ -32,7 +32,8 @@ bool memfunc_to_bool(void (Foo::*func)(int)) {
// CIR-BEFORE: %{{.*}} = cir.cast member_ptr_to_bool %{{.*}} : !cir.method<!cir.func<(!cir.ptr<!rec_Foo>, !s32i)> in !rec_Foo> -> !cir.bool
// CIR-AFTER: cir.func {{.*}} @_Z15memfunc_to_boolM3FooFviE
-// CIR-AFTER: cir.store %{{.*}}, %[[FUNC_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-X86: cir.copy %{{.*}} align(8) to %[[FUNC_ADDR:.*]] align(8) : !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-ARM: cir.store %{{.*}}, %[[FUNC_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CIR-AFTER: %[[FUNC:.*]] = cir.load{{.*}} %[[FUNC_ADDR]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
// CIR-AFTER: %[[NULL_VAL:.*]] = cir.const #cir.int<0> : !s64i
// CIR-AFTER: %[[FUNC_PTR:.*]] = cir.extract_member %[[FUNC]][0] : !rec_anon_struct -> !s64i
@@ -49,7 +50,8 @@ bool memfunc_to_bool(void (Foo::*func)(int)) {
// LLVM-X86: define dso_local noundef zeroext i1 @_Z15memfunc_to_boolM3FooFviE(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
// LLVM-ARM: define dso_local noundef i1 @_Z15memfunc_to_boolM3FooFviE({ i64, i64 } %{{[^,)]+}})
-// LLVM: store { i64, i64 } %{{.*}}, ptr %[[FUNC_ADDR:.*]], align 8
+// LLVM-X86: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[FUNC_ADDR:[^,]+]], ptr align 8 %{{.*}}, i64 16, i1 false)
+// LLVM-ARM: store { i64, i64 } %{{.*}}, ptr %[[FUNC_ADDR:.*]], align 8
// LLVM: %[[FUNC:.*]] = load { i64, i64 }, ptr %[[FUNC_ADDR]]
// LLVM: %[[FUNC_PTR:.*]] = extractvalue { i64, i64 } %[[FUNC]], 0
// LLVM: %[[BOOL_VAL:.*]] = icmp ne i64 %[[FUNC_PTR]], 0
@@ -87,7 +89,8 @@ auto memfunc_reinterpret(void (Foo::*func)(int)) -> void (Bar::*)() {
// CIR-BEFORE: %{{.*}} = cir.cast bitcast %{{.*}} : !cir.method<!cir.func<(!cir.ptr<!rec_Foo>, !s32i)> in !rec_Foo> -> !cir.method<!cir.func<(!cir.ptr<!rec_Bar>)> in !rec_Bar>
// CIR-AFTER: cir.func {{.*}} @_Z19memfunc_reinterpretM3FooFviE
-// CIR-AFTER: cir.store %{{.*}}, %[[FUNC_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-X86: cir.copy %{{.*}} align(8) to %[[FUNC_ADDR:.*]] align(8) : !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-ARM: cir.store %{{.*}}, %[[FUNC_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CIR-AFTER: %[[FUNC:.*]] = cir.load{{.*}} %[[FUNC_ADDR]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
// CIR-AFTER: cir.store %[[FUNC]], %[[RET_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CIR-AFTER: %[[RET:.*]] = cir.load{{.*}} %[[RET_ADDR]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
@@ -95,7 +98,8 @@ auto memfunc_reinterpret(void (Foo::*func)(int)) -> void (Bar::*)() {
// LLVM-X86: define dso_local { i64, i64 } @_Z19memfunc_reinterpretM3FooFviE(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
// LLVM-ARM: define dso_local { i64, i64 } @_Z19memfunc_reinterpretM3FooFviE({ i64, i64 } %{{[^,)]+}})
-// LLVM: store { i64, i64 } %{{.*}}, ptr %[[FUNC_ADDR:.*]], align 8
+// LLVM-X86: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[FUNC_ADDR:[^,]+]], ptr align 8 %{{.*}}, i64 16, i1 false)
+// LLVM-ARM: store { i64, i64 } %{{.*}}, ptr %[[FUNC_ADDR:.*]], align 8
// LLVM: %[[FUNC:.*]] = load { i64, i64 }, ptr %[[FUNC_ADDR]]
// LLVM: store { i64, i64 } %[[FUNC]], ptr %[[RET_ADDR:.*]]
// LLVM: %[[RET:.*]] = load { i64, i64 }, ptr %[[RET_ADDR]]
@@ -140,7 +144,8 @@ DerivedMemFunc base_to_derived_zero_offset(Base1MemFunc ptr) {
// CIR-AFTER: cir.func {{.*}} @_Z27base_to_derived_zero_offsetM5Base1FviE
// CIR-AFTER: %[[PTR:.*]] = cir.alloca "ptr" {{.*}} init : !cir.ptr<!rec_anon_struct>
// CIR-AFTER: %[[RET:.*]] = cir.alloca "__retval" {{.*}} : !cir.ptr<!rec_anon_struct>
-// CIR-AFTER: cir.store %{{.*}}, %[[PTR]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-X86: cir.copy %{{.*}} align(8) to %[[PTR]] align(8) : !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-ARM: cir.store %{{.*}}, %[[PTR]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CIR-AFTER: %[[TMP:.*]] = cir.load{{.*}} %[[PTR]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
// CIR-AFTER: cir.store %[[TMP]], %[[RET]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CIR-AFTER: %[[RET_VAL:.*]] = cir.load %[[RET]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
@@ -148,7 +153,8 @@ DerivedMemFunc base_to_derived_zero_offset(Base1MemFunc ptr) {
// LLVM-X86: define dso_local { i64, i64 } @_Z27base_to_derived_zero_offsetM5Base1FviE(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
// LLVM-ARM: define dso_local { i64, i64 } @_Z27base_to_derived_zero_offsetM5Base1FviE({ i64, i64 } %{{[^,)]+}})
-// LLVM: store { i64, i64 } %{{.*}}, ptr %[[ARG_ADDR:.*]], align 8
+// LLVM-X86: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[ARG_ADDR:[^,]+]], ptr align 8 %{{.*}}, i64 16, i1 false)
+// LLVM-ARM: store { i64, i64 } %{{.*}}, ptr %[[ARG_ADDR:.*]], align 8
// LLVM: %[[TMP:.*]] = load { i64, i64 }, ptr %[[ARG_ADDR]]
// LLVM: store { i64, i64 } %[[TMP]], ptr %[[RET_ADDR:.*]], align 8
// LLVM: %[[RET:.*]] = load { i64, i64 }, ptr %[[RET_ADDR]]
@@ -171,7 +177,8 @@ DerivedMemFunc base_to_derived(Base2MemFunc ptr) {
// CIR-BEFORE: %{{.*}} = cir.derived_method %[[PTR]][16] : !cir.method<!cir.func<(!cir.ptr<!rec_Base2>, !s32i)> in !rec_Base2> -> !cir.method<!cir.func<(!cir.ptr<!rec_Derived>, !s32i)> in !rec_Derived>
// CIR-AFTER: cir.func {{.*}} @_Z15base_to_derivedM5Base2FviE
-// CIR-AFTER: cir.store %{{.*}}, %[[PTR_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-X86: cir.copy %{{.*}} align(8) to %[[PTR_ADDR:.*]] align(8) : !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-ARM: cir.store %{{.*}}, %[[PTR_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CIR-AFTER: %[[PTR:.*]] = cir.load{{.*}} %[[PTR_ADDR]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
// CIR-AFTER: %[[OFFSET:.*]] = cir.extract_member %[[PTR]][1] : !rec_anon_struct -> !s64i
// CIR-AFTER-X86: %[[OFFSET_ADJ:.*]] = cir.const #cir.int<16> : !s64i
@@ -181,7 +188,8 @@ DerivedMemFunc base_to_derived(Base2MemFunc ptr) {
// LLVM-X86: define dso_local { i64, i64 } @_Z15base_to_derivedM5Base2FviE(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
// LLVM-ARM: define dso_local { i64, i64 } @_Z15base_to_derivedM5Base2FviE({ i64, i64 } %{{[^,)]+}})
-// LLVM: store { i64, i64 } %{{.*}}, ptr %[[ARG_ADDR:.*]], align 8
+// LLVM-X86: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[ARG_ADDR:[^,]+]], ptr align 8 %{{.*}}, i64 16, i1 false)
+// LLVM-ARM: store { i64, i64 } %{{.*}}, ptr %[[ARG_ADDR:.*]], align 8
// LLVM: %[[ARG:.*]] = load { i64, i64 }, ptr %[[ARG_ADDR]]
// LLVM: %[[ADJ:.*]] = extractvalue { i64, i64 } %[[ARG]], 1
// LLVM-X86: %[[ADJ_ADJ:.*]] = add nsw i64 %[[ADJ]], 16
@@ -209,7 +217,8 @@ Base1MemFunc derived_to_base_zero_offset(DerivedMemFunc ptr) {
// CIR-AFTER: cir.func {{.*}} @_Z27derived_to_base_zero_offsetM7DerivedFviE
// CIR-AFTER: %[[PTR:.*]] = cir.alloca "ptr" {{.*}} init : !cir.ptr<!rec_anon_struct>
// CIR-AFTER: %[[RET:.*]] = cir.alloca "__retval" {{.*}} : !cir.ptr<!rec_anon_struct>
-// CIR-AFTER: cir.store %{{.*}}, %[[PTR]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-X86: cir.copy %{{.*}} align(8) to %[[PTR]] align(8) : !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-ARM: cir.store %{{.*}}, %[[PTR]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CIR-AFTER: %[[TMP:.*]] = cir.load{{.*}} %[[PTR]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
// CIR-AFTER: cir.store %[[TMP]], %[[RET]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CIR-AFTER: %[[RET_VAL:.*]] = cir.load %[[RET]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
@@ -217,7 +226,8 @@ Base1MemFunc derived_to_base_zero_offset(DerivedMemFunc ptr) {
// LLVM-X86: define dso_local { i64, i64 } @_Z27derived_to_base_zero_offsetM7DerivedFviE(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
// LLVM-ARM: define dso_local { i64, i64 } @_Z27derived_to_base_zero_offsetM7DerivedFviE({ i64, i64 } %{{[^,)]+}})
-// LLVM: store { i64, i64 } %{{.*}}, ptr %[[ARG_ADDR:.*]], align 8
+// LLVM-X86: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[ARG_ADDR:[^,]+]], ptr align 8 %{{.*}}, i64 16, i1 false)
+// LLVM-ARM: store { i64, i64 } %{{.*}}, ptr %[[ARG_ADDR:.*]], align 8
// LLVM: %[[TMP:.*]] = load { i64, i64 }, ptr %[[ARG_ADDR]]
// LLVM: store { i64, i64 } %[[TMP]], ptr %[[RET_ADDR:.*]], align 8
// LLVM: %[[RET:.*]] = load { i64, i64 }, ptr %[[RET_ADDR]]
@@ -244,7 +254,8 @@ Base2MemFunc derived_to_base(DerivedMemFunc ptr) {
// CIR-BEFORE: %{{.*}} = cir.base_method %[[PTR]][16] : !cir.method<!cir.func<(!cir.ptr<!rec_Derived>, !s32i)> in !rec_Derived> -> !cir.method<!cir.func<(!cir.ptr<!rec_Base2>, !s32i)> in !rec_Base2>
// CIR-AFTER: cir.func {{.*}} @_Z15derived_to_baseM7DerivedFviE
-// CIR-AFTER: cir.store %{{.*}}, %[[PTR_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-X86: cir.copy %{{.*}} align(8) to %[[PTR_ADDR:.*]] align(8) : !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-ARM: cir.store %{{.*}}, %[[PTR_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CIR-AFTER: %[[PTR:.*]] = cir.load{{.*}} %[[PTR_ADDR]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
// CIR-AFTER: %[[OFFSET:.*]] = cir.extract_member %[[PTR]][1] : !rec_anon_struct -> !s64i
// CIR-AFTER-X86: %[[OFFSET_ADJ:.*]] = cir.const #cir.int<16> : !s64i
@@ -254,7 +265,8 @@ Base2MemFunc derived_to_base(DerivedMemFunc ptr) {
// LLVM-X86: define dso_local { i64, i64 } @_Z15derived_to_baseM7DerivedFviE(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
// LLVM-ARM: define dso_local { i64, i64 } @_Z15derived_to_baseM7DerivedFviE({ i64, i64 } %{{[^,)]+}})
-// LLVM: store { i64, i64 } %{{.*}}, ptr %[[ARG_ADDR:.*]], align 8
+// LLVM-X86: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[ARG_ADDR:[^,]+]], ptr align 8 %{{.*}}, i64 16, i1 false)
+// LLVM-ARM: store { i64, i64 } %{{.*}}, ptr %[[ARG_ADDR:.*]], align 8
// LLVM: %[[ARG:.*]] = load { i64, i64 }, ptr %[[ARG_ADDR]]
// LLVM: %[[ADJ:.*]] = extractvalue { i64, i64 } %[[ARG]], 1
// LLVM-X86: %[[ADJ_ADJ:.*]] = sub nsw i64 %[[ADJ]], 16
diff --git a/clang/test/CIR/CodeGen/pointer-to-member-func-cmp.cpp b/clang/test/CIR/CodeGen/pointer-to-member-func-cmp.cpp
index 65c68a73b78ca0..ab9e1bf76606a8 100644
--- a/clang/test/CIR/CodeGen/pointer-to-member-func-cmp.cpp
+++ b/clang/test/CIR/CodeGen/pointer-to-member-func-cmp.cpp
@@ -31,8 +31,10 @@ bool cmp_eq(void (Foo::*lhs)(int), void (Foo::*rhs)(int)) {
// CIR-BEFORE: cir.store %[[CMP]], %{{.*}} : !cir.bool, !cir.ptr<!cir.bool>
// CIR-AFTER: @_Z6cmp_eqM3FooFviES1_
-// CIR-AFTER: cir.store %{{.*}}, %[[LHS_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
-// CIR-AFTER: cir.store %{{.*}}, %[[RHS_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-X86: cir.copy %{{.*}} align(8) to %[[LHS_ADDR:[^ ]+]] align(8) : !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-X86: cir.copy %{{.*}} align(8) to %[[RHS_ADDR:[^ ]+]] align(8) : !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-ARM: cir.store %{{.*}}, %[[LHS_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-ARM: cir.store %{{.*}}, %[[RHS_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CIR-AFTER: %[[LHS:.*]] = cir.load{{.*}} %[[LHS_ADDR]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
// CIR-AFTER: %[[RHS:.*]] = cir.load{{.*}} %[[RHS_ADDR]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
// CIR-AFTER: %[[NULL:.*]] = cir.const #cir.int<0> : !s64i
@@ -54,8 +56,10 @@ bool cmp_eq(void (Foo::*lhs)(int), void (Foo::*rhs)(int)) {
// LLVM-X86: define dso_local noundef zeroext i1 @_Z6cmp_eqM3FooFviES1_(i64 %{{[^,)]+}}, i64 %{{[^,)]+}}, i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
// LLVM-ARM: define dso_local noundef i1 @_Z6cmp_eqM3FooFviES1_({ i64, i64 } %{{[^,)]+}}, { i64, i64 } %{{[^,)]+}})
-// LLVM: store { i64, i64 } %{{.*}}, ptr %[[LHS_ADDR:.*]], align 8
-// LLVM: store { i64, i64 } %{{.*}}, ptr %[[RHS_ADDR:.*]], align 8
+// LLVM-X86: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[LHS_ADDR:[^,]+]], ptr align 8 %{{.*}}, i64 16, i1 false)
+// LLVM-X86: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[RHS_ADDR:[^,]+]], ptr align 8 %{{.*}}, i64 16, i1 false)
+// LLVM-ARM: store { i64, i64 } %{{.*}}, ptr %[[LHS_ADDR:.*]], align 8
+// LLVM-ARM: store { i64, i64 } %{{.*}}, ptr %[[RHS_ADDR:.*]], align 8
// LLVM: %[[LHS:.*]] = load { i64, i64 }, ptr %[[LHS_ADDR]]
// LLVM: %[[RHS:.*]] = load { i64, i64 }, ptr %[[RHS_ADDR]]
// LLVM: %[[LHS_PTR:.*]] = extractvalue { i64, i64 } %[[LHS]], 0
@@ -107,8 +111,10 @@ bool cmp_ne(void (Foo::*lhs)(int), void (Foo::*rhs)(int)) {
// CIR-BEFORE: cir.store %[[CMP]], %{{.*}} : !cir.bool, !cir.ptr<!cir.bool>
// CIR-AFTER: cir.func {{.*}} @_Z6cmp_neM3FooFviES1_
-// CIR-AFTER: cir.store %{{.*}}, %[[LHS_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
-// CIR-AFTER: cir.store %{{.*}}, %[[RHS_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-X86: cir.copy %{{.*}} align(8) to %[[LHS_ADDR:[^ ]+]] align(8) : !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-X86: cir.copy %{{.*}} align(8) to %[[RHS_ADDR:[^ ]+]] align(8) : !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-ARM: cir.store %{{.*}}, %[[LHS_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CIR-AFTER-ARM: cir.store %{{.*}}, %[[RHS_ADDR:.*]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CIR-AFTER: %[[LHS:.*]] = cir.load{{.*}} %[[LHS_ADDR]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
// CIR-AFTER: %[[RHS:.*]] = cir.load{{.*}} %[[RHS_ADDR]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
// CIR-AFTER: %[[NULL:.*]] = cir.const #cir.int<0> : !s64i
@@ -130,8 +136,10 @@ bool cmp_ne(void (Foo::*lhs)(int), void (Foo::*rhs)(int)) {
// LLVM-X86: define dso_local noundef zeroext i1 @_Z6cmp_neM3FooFviES1_(i64 %{{[^,)]+}}, i64 %{{[^,)]+}}, i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
// LLVM-ARM: define dso_local noundef i1 @_Z6cmp_neM3FooFviES1_({ i64, i64 } %{{[^,)]+}}, { i64, i64 } %{{[^,)]+}})
-// LLVM: store { i64, i64 } %{{.*}}, ptr %[[LHS_ADDR:.*]], align 8
-// LLVM: store { i64, i64 } %{{.*}}, ptr %[[RHS_ADDR:.*]], align 8
+// LLVM-X86: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[LHS_ADDR:[^,]+]], ptr align 8 %{{.*}}, i64 16, i1 false)
+// LLVM-X86: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[RHS_ADDR:[^,]+]], ptr align 8 %{{.*}}, i64 16, i1 false)
+// LLVM-ARM: store { i64, i64 } %{{.*}}, ptr %[[LHS_ADDR:.*]], align 8
+// LLVM-ARM: store { i64, i64 } %{{.*}}, ptr %[[RHS_ADDR:.*]], align 8
// LLVM: %[[LHS:.*]] = load { i64, i64 }, ptr %[[LHS_ADDR]]
// LLVM: %[[RHS:.*]] = load { i64, i64 }, ptr %[[RHS_ADDR]]
// LLVM: %[[LHS_PTR:.*]] = extractvalue { i64, i64 } %[[LHS]], 0
diff --git a/clang/test/CIR/CodeGen/struct.cpp b/clang/test/CIR/CodeGen/struct.cpp
index 72ad4558c24301..0ab59692d6c2e8 100644
--- a/clang/test/CIR/CodeGen/struct.cpp
+++ b/clang/test/CIR/CodeGen/struct.cpp
@@ -327,11 +327,16 @@ void struct_with_const_member_expr() {
void function_arg_with_default_value(CompleteS a = {1, 2}) {}
-// CIR: %[[ARG_ADDR:.*]] = cir.alloca "a" {{.*}} init : !cir.ptr<!rec_CompleteS>
-// CIR: cir.store %{{.*}}, %[[ARG_ADDR]] : !rec_CompleteS, !cir.ptr<!rec_CompleteS>
-
+// CIR: %[[COERCE:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+// CIR: cir.store %arg0, %[[COERCE]] : !u64i, !cir.ptr<!u64i>
+// CIR: %[[COERCE_REC:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_CompleteS>
+// CIR: %[[ARG_ADDR:.*]] = cir.alloca "a" align(4) init : !cir.ptr<!rec_CompleteS>
+// CIR: cir.copy %[[COERCE_REC]] align(8) to %[[ARG_ADDR]] align(4) : !cir.ptr<!rec_CompleteS>
+
+// LLVM: %[[COERCE:.*]] = alloca i64, align 8
+// LLVM: store i64 %{{.*}}, ptr %[[COERCE]], align 8
// LLVM: %[[ARG_ADDR:.*]] = alloca %struct.CompleteS, align 4
-// LLVM: store %struct.CompleteS %{{.*}}, ptr %[[ARG_ADDR]], align 4
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[ARG_ADDR]], ptr align 8 %[[COERCE]], i64 8, i1 false)
// OGCG: %[[ARG_ADDR:.*]] = alloca %struct.CompleteS, align 4
// OGCG: store i64 %{{.*}}, ptr %[[ARG_ADDR]], align 4
@@ -340,7 +345,6 @@ void calling_function_with_default_values() {
function_arg_with_default_value();
}
-// CIR: %[[COERCE:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!rec_CompleteS>
// CIR: %[[AGG_ADDR:.*]] = cir.alloca "agg.tmp0" {{.*}} : !cir.ptr<!rec_CompleteS>
// CIR: %[[ELEM_0_PTR:.*]] = cir.get_member %[[AGG_ADDR]][0] {name = "a"} : !cir.ptr<!rec_CompleteS> -> !cir.ptr<!s32i>
// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
@@ -348,21 +352,16 @@ void calling_function_with_default_values() {
// CIR: %[[ELEM_1_PTR:.*]] = cir.get_member %[[AGG_ADDR]][1] {name = "b"} : !cir.ptr<!rec_CompleteS> -> !cir.ptr<!s8i>
// CIR: %[[CONST_2:.*]] = cir.const #cir.int<2> : !s8i
// CIR: cir.store{{.*}} %[[CONST_2]], %[[ELEM_1_PTR]] : !s8i, !cir.ptr<!s8i>
-// CIR: %[[TMP_AGG:.*]] = cir.load{{.*}} %[[AGG_ADDR]] : !cir.ptr<!rec_CompleteS>, !rec_CompleteS
-// CIR: cir.store %[[TMP_AGG]], %[[COERCE]] : !rec_CompleteS, !cir.ptr<!rec_CompleteS>
-// CIR: %[[COERCE_PTR:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!rec_CompleteS> -> !cir.ptr<!u64i>
-// CIR: %[[ARG:.*]] = cir.load %[[COERCE_PTR]] : !cir.ptr<!u64i>, !u64i
+// CIR: %[[AGG_PTR:.*]] = cir.cast bitcast %[[AGG_ADDR]] : !cir.ptr<!rec_CompleteS> -> !cir.ptr<!u64i>
+// CIR: %[[ARG:.*]] = cir.load align(4) %[[AGG_PTR]] : !cir.ptr<!u64i>, !u64i
// CIR: cir.call @_Z31function_arg_with_default_value9CompleteS(%[[ARG]]) : (!u64i) -> ()
-// LLVM: %[[COERCE:.*]] = alloca %struct.CompleteS, align 8
// LLVM: %[[AGG_ADDR:.*]] = alloca %struct.CompleteS, align 4
// LLVM: %[[ELEM_0_PTR:.*]] = getelementptr inbounds nuw %struct.CompleteS, ptr %[[AGG_ADDR]], i32 0, i32 0
// LLVM: store i32 1, ptr %[[ELEM_0_PTR]], align 4
// LLVM: %[[ELEM_1_PTR:.*]] = getelementptr inbounds nuw %struct.CompleteS, ptr %[[AGG_ADDR]], i32 0, i32 1
// LLVM: store i8 2, ptr %[[ELEM_1_PTR]], align 4
-// LLVM: %[[TMP_AGG:.*]] = load %struct.CompleteS, ptr %[[AGG_ADDR]], align 4
-// LLVM: store %struct.CompleteS %[[TMP_AGG]], ptr %[[COERCE]], align 4
-// LLVM: %[[ARG:.*]] = load i64, ptr %[[COERCE]], align 8
+// LLVM: %[[ARG:.*]] = load i64, ptr %[[AGG_ADDR]], align 4
// LLVM: call void @_Z31function_arg_with_default_value9CompleteS(i64 %[[ARG]])
// OGCG: %[[AGG_ADDR:.*]] = alloca %struct.CompleteS, align 4
diff --git a/clang/test/CIR/CodeGen/ternary-throw.cpp b/clang/test/CIR/CodeGen/ternary-throw.cpp
index ec617bea1c930b..f995d8aa4461be 100644
--- a/clang/test/CIR/CodeGen/ternary-throw.cpp
+++ b/clang/test/CIR/CodeGen/ternary-throw.cpp
@@ -1,9 +1,9 @@
// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -fexceptions -fcxx-exceptions -emit-cir %s -o %t.cir
// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -fexceptions -fcxx-exceptions -emit-llvm %s -o %t-cir.ll
-// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVMCIR --input-file=%t-cir.ll %s
// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fexceptions -fcxx-exceptions -emit-llvm %s -o %t.ll
-// RUN: FileCheck --input-file=%t.ll %s --check-prefix=OGCG
+// RUN: FileCheck --input-file=%t.ll %s --check-prefixes=LLVM,OGCG
const int& test_cond_throw_false(bool flag) {
const int a = 10;
@@ -26,30 +26,30 @@ const int& test_cond_throw_false(bool flag) {
// CIR: cir.unreachable
// CIR-NEXT: }) : (!cir.bool) -> !cir.ptr<!s32i>
-// LLVM-LABEL: define{{.*}} ptr @_Z21test_cond_throw_falseb(
-// LLVM: %[[FLAG_ALLOCA:.*]] = alloca i8
-// LLVM: %[[RET_ALLOCA:.*]] = alloca ptr
-// LLVM: %[[A_ALLOCA:.*]] = alloca i32
-// LLVM: %[[ZEXT:.*]] = zext i1 %{{.*}} to i8
-// LLVM: store i8 %[[ZEXT]], ptr %[[FLAG_ALLOCA]]
-// LLVM: store i32 10, ptr %[[A_ALLOCA]]
-// LLVM: %[[LOAD:.*]] = load i8, ptr %[[FLAG_ALLOCA]]
-// LLVM: %[[BOOL:.*]] = trunc i8 %[[LOAD]] to i1
-// LLVM: br i1 %[[BOOL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// LLVM: [[TRUE_BB]]:
-// LLVM: br label %[[PHI_BB:.*]]
-// LLVM: [[FALSE_BB]]:
-// LLVM: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
-// LLVM: store i32 0, ptr %[[EXC]]
-// LLVM: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
-// LLVM: unreachable
-// LLVM: [[PHI_BB]]:
-// LLVM: %[[PHI:.*]] = phi ptr [ %[[A_ALLOCA]], %[[TRUE_BB]] ]
-// LLVM: br label %[[CONT_BB:.*]]
-// LLVM: [[CONT_BB]]:
-// LLVM: store ptr %[[PHI]], ptr %[[RET_ALLOCA]]
-// LLVM: %[[RET:.*]] = load ptr, ptr %[[RET_ALLOCA]]
-// LLVM: ret ptr %[[RET]]
+// LLVMCIR-LABEL: define{{.*}} ptr @_Z21test_cond_throw_falseb(
+// LLVMCIR: %[[FLAG_ALLOCA:.*]] = alloca i8
+// LLVMCIR: %[[RET_ALLOCA:.*]] = alloca ptr
+// LLVMCIR: %[[A_ALLOCA:.*]] = alloca i32
+// LLVMCIR: %[[ZEXT:.*]] = zext i1 %{{.*}} to i8
+// LLVMCIR: store i8 %[[ZEXT]], ptr %[[FLAG_ALLOCA]]
+// LLVMCIR: store i32 10, ptr %[[A_ALLOCA]]
+// LLVMCIR: %[[LOAD:.*]] = load i8, ptr %[[FLAG_ALLOCA]]
+// LLVMCIR: %[[BOOL:.*]] = trunc i8 %[[LOAD]] to i1
+// LLVMCIR: br i1 %[[BOOL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVMCIR: [[TRUE_BB]]:
+// LLVMCIR: br label %[[PHI_BB:.*]]
+// LLVMCIR: [[FALSE_BB]]:
+// LLVMCIR: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
+// LLVMCIR: store i32 0, ptr %[[EXC]]
+// LLVMCIR: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
+// LLVMCIR: unreachable
+// LLVMCIR: [[PHI_BB]]:
+// LLVMCIR: %[[PHI:.*]] = phi ptr [ %[[A_ALLOCA]], %[[TRUE_BB]] ]
+// LLVMCIR: br label %[[CONT_BB:.*]]
+// LLVMCIR: [[CONT_BB]]:
+// LLVMCIR: store ptr %[[PHI]], ptr %[[RET_ALLOCA]]
+// LLVMCIR: %[[RET:.*]] = load ptr, ptr %[[RET_ALLOCA]]
+// LLVMCIR: ret ptr %[[RET]]
// OGCG-LABEL: define{{.*}} ptr @_Z21test_cond_throw_falseb(
// OGCG: %{{.*}} = alloca i8
@@ -89,30 +89,30 @@ const int& test_cond_throw_true(bool flag) {
// CIR: cir.yield %[[A]] : !cir.ptr<!s32i>
// CIR-NEXT: }) : (!cir.bool) -> !cir.ptr<!s32i>
-// LLVM-LABEL: define{{.*}} ptr @_Z20test_cond_throw_trueb(
-// LLVM: %[[FLAG_ALLOCA:.*]] = alloca i8
-// LLVM: %[[RET_ALLOCA:.*]] = alloca ptr
-// LLVM: %[[A_ALLOCA:.*]] = alloca i32
-// LLVM: %[[ZEXT:.*]] = zext i1 %{{.*}} to i8
-// LLVM: store i8 %[[ZEXT]], ptr %[[FLAG_ALLOCA]]
-// LLVM: store i32 10, ptr %[[A_ALLOCA]]
-// LLVM: %[[LOAD:.*]] = load i8, ptr %[[FLAG_ALLOCA]]
-// LLVM: %[[BOOL:.*]] = trunc i8 %[[LOAD]] to i1
-// LLVM: br i1 %[[BOOL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// LLVM: [[TRUE_BB]]:
-// LLVM: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
-// LLVM: store i32 0, ptr %[[EXC]]
-// LLVM: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
-// LLVM: unreachable
-// LLVM: [[FALSE_BB]]:
-// LLVM: br label %[[PHI_BB:.*]]
-// LLVM: [[PHI_BB]]:
-// LLVM: %[[PHI:.*]] = phi ptr [ %[[A_ALLOCA]], %[[FALSE_BB]] ]
-// LLVM: br label %[[CONT_BB:.*]]
-// LLVM: [[CONT_BB]]:
-// LLVM: store ptr %[[PHI]], ptr %[[RET_ALLOCA]]
-// LLVM: %[[RET:.*]] = load ptr, ptr %[[RET_ALLOCA]]
-// LLVM: ret ptr %[[RET]]
+// LLVMCIR-LABEL: define{{.*}} ptr @_Z20test_cond_throw_trueb(
+// LLVMCIR: %[[FLAG_ALLOCA:.*]] = alloca i8
+// LLVMCIR: %[[RET_ALLOCA:.*]] = alloca ptr
+// LLVMCIR: %[[A_ALLOCA:.*]] = alloca i32
+// LLVMCIR: %[[ZEXT:.*]] = zext i1 %{{.*}} to i8
+// LLVMCIR: store i8 %[[ZEXT]], ptr %[[FLAG_ALLOCA]]
+// LLVMCIR: store i32 10, ptr %[[A_ALLOCA]]
+// LLVMCIR: %[[LOAD:.*]] = load i8, ptr %[[FLAG_ALLOCA]]
+// LLVMCIR: %[[BOOL:.*]] = trunc i8 %[[LOAD]] to i1
+// LLVMCIR: br i1 %[[BOOL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVMCIR: [[TRUE_BB]]:
+// LLVMCIR: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
+// LLVMCIR: store i32 0, ptr %[[EXC]]
+// LLVMCIR: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
+// LLVMCIR: unreachable
+// LLVMCIR: [[FALSE_BB]]:
+// LLVMCIR: br label %[[PHI_BB:.*]]
+// LLVMCIR: [[PHI_BB]]:
+// LLVMCIR: %[[PHI:.*]] = phi ptr [ %[[A_ALLOCA]], %[[FALSE_BB]] ]
+// LLVMCIR: br label %[[CONT_BB:.*]]
+// LLVMCIR: [[CONT_BB]]:
+// LLVMCIR: store ptr %[[PHI]], ptr %[[RET_ALLOCA]]
+// LLVMCIR: %[[RET:.*]] = load ptr, ptr %[[RET_ALLOCA]]
+// LLVMCIR: ret ptr %[[RET]]
// OGCG-LABEL: define{{.*}} ptr @_Z20test_cond_throw_trueb(
// OGCG: %{{.*}} = alloca i8
@@ -147,14 +147,14 @@ const int& test_cond_const_true_throw_false() {
// CIR: %[[RET:.*]] = cir.load
// CIR: cir.return %[[RET]] : !cir.ptr<!s32i>
-// LLVM-LABEL: define{{.*}} ptr @_Z32test_cond_const_true_throw_falsev(
-// LLVM: %[[A:.*]] = alloca i32
-// LLVM: store i32 20, ptr %[[A]]
-// LLVM-NOT: br i1
-// LLVM-NOT: __cxa_throw
-// LLVM: store ptr %[[A]]
-// LLVM: %[[RET:.*]] = load ptr
-// LLVM: ret ptr %[[RET]]
+// LLVMCIR-LABEL: define{{.*}} ptr @_Z32test_cond_const_true_throw_falsev(
+// LLVMCIR: %[[A:.*]] = alloca i32
+// LLVMCIR: store i32 20, ptr %[[A]]
+// LLVMCIR-NOT: br i1
+// LLVMCIR-NOT: __cxa_throw
+// LLVMCIR: store ptr %[[A]]
+// LLVMCIR: %[[RET:.*]] = load ptr
+// LLVMCIR: ret ptr %[[RET]]
// OGCG-LABEL: define{{.*}} ptr @_Z32test_cond_const_true_throw_falsev(
// OGCG: %[[A:.*]] = alloca i32
@@ -179,14 +179,14 @@ const int& test_cond_const_false_throw_true() {
// CIR: %[[RET:.*]] = cir.load
// CIR: cir.return %[[RET]] : !cir.ptr<!s32i>
-// LLVM-LABEL: define{{.*}} ptr @_Z32test_cond_const_false_throw_truev(
-// LLVM: %[[A:.*]] = alloca i32
-// LLVM: store i32 30, ptr %[[A]]
-// LLVM-NOT: br i1
-// LLVM-NOT: __cxa_throw
-// LLVM: store ptr %[[A]]
-// LLVM: %[[RET:.*]] = load ptr
-// LLVM: ret ptr %[[RET]]
+// LLVMCIR-LABEL: define{{.*}} ptr @_Z32test_cond_const_false_throw_truev(
+// LLVMCIR: %[[A:.*]] = alloca i32
+// LLVMCIR: store i32 30, ptr %[[A]]
+// LLVMCIR-NOT: br i1
+// LLVMCIR-NOT: __cxa_throw
+// LLVMCIR: store ptr %[[A]]
+// LLVMCIR: %[[RET:.*]] = load ptr
+// LLVMCIR: ret ptr %[[RET]]
// OGCG-LABEL: define{{.*}} ptr @_Z32test_cond_const_false_throw_truev(
// OGCG: %[[A:.*]] = alloca i32
@@ -216,18 +216,18 @@ const int &test_cond_const_true_throw_true() {
// CIR: %[[TMP_RET:.*]] = cir.load %[[RET_ADDR]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
// CIR: cir.return %[[TMP_RET]] : !cir.ptr<!s32i>
-// LLVM-LABEL: define{{.*}} ptr @_Z31test_cond_const_true_throw_truev(
-// LLVM: %[[RET_ADDR:.*]] = alloca ptr, align 8
-// LLVM: %[[A_ADDR:.*]] = alloca i32, align 4
-// LLVM: store i32 30, ptr %[[A_ADDR]], align 4
-// LLVM: %[[EXCEPTION:.*]] = call ptr @__cxa_allocate_exception(i64 4)
-// LLVM: store i32 0, ptr %[[EXCEPTION]], align 16
-// LLVM: call void @__cxa_throw(ptr %[[EXCEPTION]], ptr @_ZTIi, ptr null)
-// LLVM: unreachable
-// LLVM: [[NO_PRED_LABEL:.*]]:
-// LLVM: store ptr null, ptr %[[RET_ADDR]], align 8
-// LLVM: %[[TMP_RET:.*]] = load ptr, ptr %[[RET_ADDR]], align 8
-// LLVM: ret ptr %[[TMP_RET]]
+// LLVMCIR-LABEL: define{{.*}} ptr @_Z31test_cond_const_true_throw_truev(
+// LLVMCIR: %[[RET_ADDR:.*]] = alloca ptr, align 8
+// LLVMCIR: %[[A_ADDR:.*]] = alloca i32, align 4
+// LLVMCIR: store i32 30, ptr %[[A_ADDR]], align 4
+// LLVMCIR: %[[EXCEPTION:.*]] = call ptr @__cxa_allocate_exception(i64 4)
+// LLVMCIR: store i32 0, ptr %[[EXCEPTION]], align 16
+// LLVMCIR: call void @__cxa_throw(ptr %[[EXCEPTION]], ptr @_ZTIi, ptr null)
+// LLVMCIR: unreachable
+// LLVMCIR: [[NO_PRED_LABEL:.*]]:
+// LLVMCIR: store ptr null, ptr %[[RET_ADDR]], align 8
+// LLVMCIR: %[[TMP_RET:.*]] = load ptr, ptr %[[RET_ADDR]], align 8
+// LLVMCIR: ret ptr %[[TMP_RET]]
// OGCG-LABEL: define{{.*}} ptr @_Z31test_cond_const_true_throw_truev(
// OGCG: %[[A_ADDR:.*]] = alloca i32, align 4
@@ -262,29 +262,29 @@ int test_agg_cond_throw_false(bool flag, struct s6 a1, struct s6 a2) {
// CIR: %[[LOAD:.*]] = cir.load{{.*}} %[[F0]] : !cir.ptr<!s32i>, !s32i
// CIR: cir.return %{{.*}} : !s32i
-// LLVM-LABEL: define{{.*}} i32 @_Z25test_agg_cond_throw_falseb2s6S_(
-// LLVM: %[[FLAG_ALLOCA:.*]] = alloca i8
-// LLVM: %[[A1_ALLOCA:.*]] = alloca %struct.s6
-// LLVM: %[[A2_ALLOCA:.*]] = alloca %struct.s6
-// LLVM: %[[ZEXT:.*]] = zext i1 %{{.*}} to i8
-// LLVM: store i8 %[[ZEXT]], ptr %[[FLAG_ALLOCA]]
-// LLVM: %[[FLAG_LOAD:.*]] = load i8, ptr %[[FLAG_ALLOCA]]
-// LLVM: %[[BOOL:.*]] = trunc i8 %[[FLAG_LOAD]] to i1
-// LLVM: br i1 %[[BOOL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// LLVM: [[TRUE_BB]]:
-// LLVM: br label %[[PHI_BB:.*]]
-// LLVM: [[FALSE_BB]]:
-// LLVM: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
-// LLVM: store i32 0, ptr %[[EXC]]
-// LLVM: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
-// LLVM: unreachable
-// LLVM: [[PHI_BB]]:
-// LLVM: %[[PHI:.*]] = phi ptr [ %[[A1_ALLOCA]], %[[TRUE_BB]] ]
-// LLVM: br label %[[CONT_BB:.*]]
-// LLVM: [[CONT_BB]]:
-// LLVM: %[[F0_PTR:.*]] = getelementptr inbounds nuw %struct.s6, ptr %[[PHI]], i32 0, i32 0
-// LLVM: %[[F0_VAL:.*]] = load i32, ptr %[[F0_PTR]]
-// LLVM: ret i32 %{{.*}}
+// LLVMCIR-LABEL: define{{.*}} i32 @_Z25test_agg_cond_throw_falseb2s6S_(
+// LLVMCIR: %[[FLAG_ALLOCA:.*]] = alloca i8
+// LLVMCIR: %[[A1_ALLOCA:.*]] = alloca %struct.s6
+// LLVMCIR: %[[A2_ALLOCA:.*]] = alloca %struct.s6
+// LLVMCIR: %[[ZEXT:.*]] = zext i1 %{{.*}} to i8
+// LLVMCIR: store i8 %[[ZEXT]], ptr %[[FLAG_ALLOCA]]
+// LLVMCIR: %[[FLAG_LOAD:.*]] = load i8, ptr %[[FLAG_ALLOCA]]
+// LLVMCIR: %[[BOOL:.*]] = trunc i8 %[[FLAG_LOAD]] to i1
+// LLVMCIR: br i1 %[[BOOL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVMCIR: [[TRUE_BB]]:
+// LLVMCIR: br label %[[PHI_BB:.*]]
+// LLVMCIR: [[FALSE_BB]]:
+// LLVMCIR: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
+// LLVMCIR: store i32 0, ptr %[[EXC]]
+// LLVMCIR: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
+// LLVMCIR: unreachable
+// LLVMCIR: [[PHI_BB]]:
+// LLVMCIR: %[[PHI:.*]] = phi ptr [ %[[A1_ALLOCA]], %[[TRUE_BB]] ]
+// LLVMCIR: br label %[[CONT_BB:.*]]
+// LLVMCIR: [[CONT_BB]]:
+// LLVMCIR: %[[F0_PTR:.*]] = getelementptr inbounds nuw %struct.s6, ptr %[[PHI]], i32 0, i32 0
+// LLVMCIR: %[[F0_VAL:.*]] = load i32, ptr %[[F0_PTR]]
+// LLVMCIR: ret i32 %{{.*}}
// OGCG-LABEL: define{{.*}} i32 @_Z25test_agg_cond_throw_falseb2s6S_(
// OGCG: %[[A1:.*]] = alloca %struct.s6
@@ -327,29 +327,29 @@ int test_agg_cond_throw_true(bool flag, struct s6 a1, struct s6 a2) {
// CIR: %[[LOAD:.*]] = cir.load{{.*}} %[[F0]] : !cir.ptr<!s32i>, !s32i
// CIR: cir.return %{{.*}} : !s32i
-// LLVM-LABEL: define{{.*}} i32 @_Z24test_agg_cond_throw_trueb2s6S_(
-// LLVM: %[[FLAG_ALLOCA:.*]] = alloca i8
-// LLVM: %[[A1_ALLOCA:.*]] = alloca %struct.s6
-// LLVM: %[[A2_ALLOCA:.*]] = alloca %struct.s6
-// LLVM: %[[ZEXT:.*]] = zext i1 %{{.*}} to i8
-// LLVM: store i8 %[[ZEXT]], ptr %[[FLAG_ALLOCA]]
-// LLVM: %[[FLAG_LOAD:.*]] = load i8, ptr %[[FLAG_ALLOCA]]
-// LLVM: %[[BOOL:.*]] = trunc i8 %[[FLAG_LOAD]] to i1
-// LLVM: br i1 %[[BOOL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// LLVM: [[TRUE_BB]]:
-// LLVM: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
-// LLVM: store i32 0, ptr %[[EXC]]
-// LLVM: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
-// LLVM: unreachable
-// LLVM: [[FALSE_BB]]:
-// LLVM: br label %[[PHI_BB:.*]]
-// LLVM: [[PHI_BB]]:
-// LLVM: %[[PHI:.*]] = phi ptr [ %[[A1_ALLOCA]], %[[FALSE_BB]] ]
-// LLVM: br label %[[CONT_BB:.*]]
-// LLVM: [[CONT_BB]]:
-// LLVM: %[[F0_PTR:.*]] = getelementptr inbounds nuw %struct.s6, ptr %[[PHI]], i32 0, i32 0
-// LLVM: %[[F0_VAL:.*]] = load i32, ptr %[[F0_PTR]]
-// LLVM: ret i32 %{{.*}}
+// LLVMCIR-LABEL: define{{.*}} i32 @_Z24test_agg_cond_throw_trueb2s6S_(
+// LLVMCIR: %[[FLAG_ALLOCA:.*]] = alloca i8
+// LLVMCIR: %[[A1_ALLOCA:.*]] = alloca %struct.s6
+// LLVMCIR: %[[A2_ALLOCA:.*]] = alloca %struct.s6
+// LLVMCIR: %[[ZEXT:.*]] = zext i1 %{{.*}} to i8
+// LLVMCIR: store i8 %[[ZEXT]], ptr %[[FLAG_ALLOCA]]
+// LLVMCIR: %[[FLAG_LOAD:.*]] = load i8, ptr %[[FLAG_ALLOCA]]
+// LLVMCIR: %[[BOOL:.*]] = trunc i8 %[[FLAG_LOAD]] to i1
+// LLVMCIR: br i1 %[[BOOL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVMCIR: [[TRUE_BB]]:
+// LLVMCIR: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
+// LLVMCIR: store i32 0, ptr %[[EXC]]
+// LLVMCIR: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
+// LLVMCIR: unreachable
+// LLVMCIR: [[FALSE_BB]]:
+// LLVMCIR: br label %[[PHI_BB:.*]]
+// LLVMCIR: [[PHI_BB]]:
+// LLVMCIR: %[[PHI:.*]] = phi ptr [ %[[A1_ALLOCA]], %[[FALSE_BB]] ]
+// LLVMCIR: br label %[[CONT_BB:.*]]
+// LLVMCIR: [[CONT_BB]]:
+// LLVMCIR: %[[F0_PTR:.*]] = getelementptr inbounds nuw %struct.s6, ptr %[[PHI]], i32 0, i32 0
+// LLVMCIR: %[[F0_VAL:.*]] = load i32, ptr %[[F0_PTR]]
+// LLVMCIR: ret i32 %{{.*}}
// OGCG-LABEL: define{{.*}} i32 @_Z24test_agg_cond_throw_trueb2s6S_(
// OGCG: %[[A1:.*]] = alloca %struct.s6
@@ -383,14 +383,14 @@ const int test_agg_cond_const_true_throw_false(struct s6 a1, struct s6 a2) {
// CIR: %[[LOAD:.*]] = cir.load{{.*}} %[[F0]] : !cir.ptr<!s32i>, !s32i
// CIR: cir.return %{{.*}} : !s32i
-// LLVM-LABEL: define{{.*}} i32 @_Z36test_agg_cond_const_true_throw_false2s6S_(
-// LLVM: %[[A1_ALLOCA:.*]] = alloca %struct.s6
-// LLVM: %[[A2_ALLOCA:.*]] = alloca %struct.s6
-// LLVM-NOT: br i1
-// LLVM-NOT: __cxa_throw
-// LLVM: %[[F0_PTR:.*]] = getelementptr inbounds nuw %struct.s6, ptr %[[A1_ALLOCA]], i32 0, i32 0
-// LLVM: %[[F0_VAL:.*]] = load i32, ptr %[[F0_PTR]]
-// LLVM: ret i32 %{{.*}}
+// LLVMCIR-LABEL: define{{.*}} i32 @_Z36test_agg_cond_const_true_throw_false2s6S_(
+// LLVMCIR: %[[A1_ALLOCA:.*]] = alloca %struct.s6
+// LLVMCIR: %[[A2_ALLOCA:.*]] = alloca %struct.s6
+// LLVMCIR-NOT: br i1
+// LLVMCIR-NOT: __cxa_throw
+// LLVMCIR: %[[F0_PTR:.*]] = getelementptr inbounds nuw %struct.s6, ptr %[[A1_ALLOCA]], i32 0, i32 0
+// LLVMCIR: %[[F0_VAL:.*]] = load i32, ptr %[[F0_PTR]]
+// LLVMCIR: ret i32 %{{.*}}
// OGCG-LABEL: define{{.*}} i32 @_Z36test_agg_cond_const_true_throw_false2s6S_(
// OGCG: %[[A1:.*]] = alloca %struct.s6
@@ -424,16 +424,16 @@ const int test_agg_cond_const_true_throw_true(struct s6 a1, struct s6 a2) {
// CIR: %[[LOAD:.*]] = cir.load{{.*}} %[[F0]] : !cir.ptr<!s32i>, !s32i
// CIR: cir.return %{{.*}} : !s32i
-// LLVM-LABEL: define{{.*}} i32 @_Z35test_agg_cond_const_true_throw_true2s6S_(
-// LLVM: %[[A1_ALLOCA:.*]] = alloca %struct.s6
-// LLVM: %[[A2_ALLOCA:.*]] = alloca %struct.s6
-// LLVM: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
-// LLVM: store i32 0, ptr %[[EXC]]
-// LLVM: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
-// LLVM: unreachable
-// LLVM: [[NO_PRED:.*]]:
-// LLVM: %[[LOAD_UNDEF:.*]] = load i32, ptr null, align 1
-// LLVM: ret i32 %{{.*}}
+// LLVMCIR-LABEL: define{{.*}} i32 @_Z35test_agg_cond_const_true_throw_true2s6S_(
+// LLVMCIR: %[[A1_ALLOCA:.*]] = alloca %struct.s6
+// LLVMCIR: %[[A2_ALLOCA:.*]] = alloca %struct.s6
+// LLVMCIR: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
+// LLVMCIR: store i32 0, ptr %[[EXC]]
+// LLVMCIR: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
+// LLVMCIR: unreachable
+// LLVMCIR: [[NO_PRED:.*]]:
+// LLVMCIR: %[[LOAD_UNDEF:.*]] = load i32, ptr null, align 1
+// LLVMCIR: ret i32 %{{.*}}
// OGCG-LABEL: define{{.*}} i32 @_Z35test_agg_cond_const_true_throw_true2s6S_(
// OGCG: %[[A1:.*]] = alloca %struct.s6
@@ -464,16 +464,16 @@ const int test_agg_cond_const_false_throw_false(struct s6 a1, struct s6 a2) {
// CIR: %[[LOAD:.*]] = cir.load{{.*}} %[[F0]] : !cir.ptr<!s32i>, !s32i
// CIR: cir.return %{{.*}} : !s32i
-// LLVM-LABEL: define{{.*}} i32 @_Z37test_agg_cond_const_false_throw_false2s6S_(
-// LLVM: %[[A1_ALLOCA:.*]] = alloca %struct.s6
-// LLVM: %[[A2_ALLOCA:.*]] = alloca %struct.s6
-// LLVM: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
-// LLVM: store i32 0, ptr %[[EXC]]
-// LLVM: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
-// LLVM: unreachable
-// LLVM: [[NO_PRED:.*]]:
-// LLVM: %[[LOAD_UNDEF:.*]] = load i32, ptr null, align 1
-// LLVM: ret i32 %{{.*}}
+// LLVMCIR-LABEL: define{{.*}} i32 @_Z37test_agg_cond_const_false_throw_false2s6S_(
+// LLVMCIR: %[[A1_ALLOCA:.*]] = alloca %struct.s6
+// LLVMCIR: %[[A2_ALLOCA:.*]] = alloca %struct.s6
+// LLVMCIR: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
+// LLVMCIR: store i32 0, ptr %[[EXC]]
+// LLVMCIR: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
+// LLVMCIR: unreachable
+// LLVMCIR: [[NO_PRED:.*]]:
+// LLVMCIR: %[[LOAD_UNDEF:.*]] = load i32, ptr null, align 1
+// LLVMCIR: ret i32 %{{.*}}
// OGCG-LABEL: define{{.*}} i32 @_Z37test_agg_cond_const_false_throw_false2s6S_(
// OGCG: %[[A1:.*]] = alloca %struct.s6
@@ -499,14 +499,14 @@ const int test_agg_cond_const_false_throw_true(struct s6 a1, struct s6 a2) {
// CIR: %[[LOAD:.*]] = cir.load{{.*}} %[[F0]] : !cir.ptr<!s32i>, !s32i
// CIR: cir.return %{{.*}} : !s32i
-// LLVM-LABEL: define{{.*}} i32 @_Z36test_agg_cond_const_false_throw_true2s6S_(
-// LLVM: %[[A1_ALLOCA:.*]] = alloca %struct.s6
-// LLVM: %[[A2_ALLOCA:.*]] = alloca %struct.s6
-// LLVM-NOT: br i1
-// LLVM-NOT: __cxa_throw
-// LLVM: %[[F0_PTR:.*]] = getelementptr inbounds nuw %struct.s6, ptr %[[A1_ALLOCA]], i32 0, i32 0
-// LLVM: %[[F0_VAL:.*]] = load i32, ptr %[[F0_PTR]]
-// LLVM: ret i32 %{{.*}}
+// LLVMCIR-LABEL: define{{.*}} i32 @_Z36test_agg_cond_const_false_throw_true2s6S_(
+// LLVMCIR: %[[A1_ALLOCA:.*]] = alloca %struct.s6
+// LLVMCIR: %[[A2_ALLOCA:.*]] = alloca %struct.s6
+// LLVMCIR-NOT: br i1
+// LLVMCIR-NOT: __cxa_throw
+// LLVMCIR: %[[F0_PTR:.*]] = getelementptr inbounds nuw %struct.s6, ptr %[[A1_ALLOCA]], i32 0, i32 0
+// LLVMCIR: %[[F0_VAL:.*]] = load i32, ptr %[[F0_PTR]]
+// LLVMCIR: ret i32 %{{.*}}
// OGCG-LABEL: define{{.*}} i32 @_Z36test_agg_cond_const_false_throw_true2s6S_(
// OGCG: %[[A1:.*]] = alloca %struct.s6
@@ -551,24 +551,24 @@ void test_agg_throw_true(bool flag) {
// CIR: }
// CIR: cir.return
-// LLVM-LABEL: define{{.*}} void @_Z19test_agg_throw_trueb(
-// LLVM: %[[FLAG_ALLOCA:.*]] = alloca i8
-// LLVM: %[[A_ALLOCA:.*]] = alloca %struct.Agg
-// LLVM: %[[BOOL:.*]] = trunc i8 %{{.*}} to i1
-// LLVM: br i1 %[[BOOL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// LLVM: [[TRUE_BB]]:
-// LLVM: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
-// LLVM: store i32 0, ptr %[[EXC]]
-// LLVM: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
-// LLVM: unreachable
-// LLVM: [[FALSE_BB]]:
-// LLVM: %[[X:.*]] = getelementptr inbounds nuw %struct.Agg, ptr %[[A_ALLOCA]], i32 0, i32 0
-// LLVM: store i32 1, ptr %[[X]]
-// LLVM: %[[Y:.*]] = getelementptr inbounds nuw %struct.Agg, ptr %[[A_ALLOCA]], i32 0, i32 1
-// LLVM: store i32 2, ptr %[[Y]]
-// LLVM: br label %[[END:.*]]
-// LLVM: [[END]]:
-// LLVM: ret void
+// LLVMCIR-LABEL: define{{.*}} void @_Z19test_agg_throw_trueb(
+// LLVMCIR: %[[FLAG_ALLOCA:.*]] = alloca i8
+// LLVMCIR: %[[A_ALLOCA:.*]] = alloca %struct.Agg
+// LLVMCIR: %[[BOOL:.*]] = trunc i8 %{{.*}} to i1
+// LLVMCIR: br i1 %[[BOOL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVMCIR: [[TRUE_BB]]:
+// LLVMCIR: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
+// LLVMCIR: store i32 0, ptr %[[EXC]]
+// LLVMCIR: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
+// LLVMCIR: unreachable
+// LLVMCIR: [[FALSE_BB]]:
+// LLVMCIR: %[[X:.*]] = getelementptr inbounds nuw %struct.Agg, ptr %[[A_ALLOCA]], i32 0, i32 0
+// LLVMCIR: store i32 1, ptr %[[X]]
+// LLVMCIR: %[[Y:.*]] = getelementptr inbounds nuw %struct.Agg, ptr %[[A_ALLOCA]], i32 0, i32 1
+// LLVMCIR: store i32 2, ptr %[[Y]]
+// LLVMCIR: br label %[[END:.*]]
+// LLVMCIR: [[END]]:
+// LLVMCIR: ret void
// OGCG-LABEL: define{{.*}} void @_Z19test_agg_throw_trueb(
// OGCG: %[[A:.*]] = alloca %struct.Agg
@@ -613,24 +613,24 @@ void test_agg_throw_false(bool flag) {
// CIR-NEXT: }
// CIR: cir.return
-// LLVM-LABEL: define{{.*}} void @_Z20test_agg_throw_falseb(
-// LLVM: %[[FLAG_ALLOCA:.*]] = alloca i8
-// LLVM: %[[A_ALLOCA:.*]] = alloca %struct.Agg
-// LLVM: %[[BOOL:.*]] = trunc i8 %{{.*}} to i1
-// LLVM: br i1 %[[BOOL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// LLVM: [[TRUE_BB]]:
-// LLVM: %[[X:.*]] = getelementptr inbounds nuw %struct.Agg, ptr %[[A_ALLOCA]], i32 0, i32 0
-// LLVM: store i32 1, ptr %[[X]]
-// LLVM: %[[Y:.*]] = getelementptr inbounds nuw %struct.Agg, ptr %[[A_ALLOCA]], i32 0, i32 1
-// LLVM: store i32 2, ptr %[[Y]]
-// LLVM: br label %[[END:.*]]
-// LLVM: [[FALSE_BB]]:
-// LLVM: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
-// LLVM: store i32 0, ptr %[[EXC]]
-// LLVM: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
-// LLVM: unreachable
-// LLVM: [[END]]:
-// LLVM: ret void
+// LLVMCIR-LABEL: define{{.*}} void @_Z20test_agg_throw_falseb(
+// LLVMCIR: %[[FLAG_ALLOCA:.*]] = alloca i8
+// LLVMCIR: %[[A_ALLOCA:.*]] = alloca %struct.Agg
+// LLVMCIR: %[[BOOL:.*]] = trunc i8 %{{.*}} to i1
+// LLVMCIR: br i1 %[[BOOL]], label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVMCIR: [[TRUE_BB]]:
+// LLVMCIR: %[[X:.*]] = getelementptr inbounds nuw %struct.Agg, ptr %[[A_ALLOCA]], i32 0, i32 0
+// LLVMCIR: store i32 1, ptr %[[X]]
+// LLVMCIR: %[[Y:.*]] = getelementptr inbounds nuw %struct.Agg, ptr %[[A_ALLOCA]], i32 0, i32 1
+// LLVMCIR: store i32 2, ptr %[[Y]]
+// LLVMCIR: br label %[[END:.*]]
+// LLVMCIR: [[FALSE_BB]]:
+// LLVMCIR: %[[EXC:.*]] = call{{.*}} ptr @__cxa_allocate_exception
+// LLVMCIR: store i32 0, ptr %[[EXC]]
+// LLVMCIR: call void @__cxa_throw(ptr %[[EXC]], ptr @_ZTIi
+// LLVMCIR: unreachable
+// LLVMCIR: [[END]]:
+// LLVMCIR: ret void
// OGCG-LABEL: define{{.*}} void @_Z20test_agg_throw_falseb(
// OGCG: %[[A:.*]] = alloca %struct.Agg
@@ -666,14 +666,14 @@ int test_scalar_throw_true(bool flag, int x) {
// CIR: cir.yield %[[X]] : !s32i
// CIR: }) : (!cir.bool) -> !s32i
-// LLVM-LABEL: define{{.*}} i32 @_Z22test_scalar_throw_truebi(
-// LLVM: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// LLVM: [[TRUE_BB]]:
-// LLVM: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
-// LLVM: unreachable
-// LLVM: [[FALSE_BB]]:
-// LLVM: %{{.*}} = load i32
-// LLVM: ret i32
+// LLVMCIR-LABEL: define{{.*}} i32 @_Z22test_scalar_throw_truebi(
+// LLVMCIR: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVMCIR: [[TRUE_BB]]:
+// LLVMCIR: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
+// LLVMCIR: unreachable
+// LLVMCIR: [[FALSE_BB]]:
+// LLVMCIR: %{{.*}} = load i32
+// LLVMCIR: ret i32
// OGCG-LABEL: define{{.*}} i32 @_Z22test_scalar_throw_truebi(
// OGCG: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
@@ -697,11 +697,11 @@ int test_scalar_throw_false(bool flag, int x) {
// CIR: cir.unreachable
// CIR-NEXT: }) : (!cir.bool) -> !s32i
-// LLVM-LABEL: define{{.*}} i32 @_Z23test_scalar_throw_falsebi(
-// LLVM: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// LLVM: [[FALSE_BB]]:
-// LLVM: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
-// LLVM: unreachable
+// LLVMCIR-LABEL: define{{.*}} i32 @_Z23test_scalar_throw_falsebi(
+// LLVMCIR: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVMCIR: [[FALSE_BB]]:
+// LLVMCIR: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
+// LLVMCIR: unreachable
// OGCG-LABEL: define{{.*}} i32 @_Z23test_scalar_throw_falsebi(
// OGCG: br i1 %{{.*}}, label %{{.*}}, label %[[FALSE_BB:.*]]
@@ -731,15 +731,6 @@ void test_both_throw(bool flag) {
// LLVM: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
// LLVM: unreachable
-// OGCG-LABEL: define{{.*}} void @_Z15test_both_throwb(
-// OGCG: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// OGCG: [[TRUE_BB]]:
-// OGCG: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
-// OGCG: unreachable
-// OGCG: [[FALSE_BB]]:
-// OGCG: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
-// OGCG: unreachable
-
// The surviving arm of a glvalue conditional may compute its pointer inside
// the ternary region (e.g. loading a reference parameter); the result must be
// addressed through the cir.ternary result, which the region's yield carries
@@ -762,19 +753,19 @@ int &test_ref_cond_throw(bool c, int &x) {
// CIR-NEXT: }) : (!cir.bool) -> !cir.ptr<!s32i>
// CIR: cir.store %[[RES]], %[[RET_ADDR]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
-// LLVM-LABEL: define{{.*}} ptr @_Z19test_ref_cond_throwbRi(
-// LLVM: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// LLVM: [[TRUE_BB]]:
-// LLVM: %[[X_PTR:.*]] = load ptr, ptr %{{.*}}
-// LLVM: br label %[[PHI_BB:.*]]
-// LLVM: [[FALSE_BB]]:
-// LLVM: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
-// LLVM: unreachable
-// LLVM: [[PHI_BB]]:
-// LLVM: %[[PHI:.*]] = phi ptr [ %[[X_PTR]], %[[TRUE_BB]] ]
-// LLVM: store ptr %[[PHI]], ptr %[[RET_ALLOCA:.*]], align 8
-// LLVM: %[[RET:.*]] = load ptr, ptr %[[RET_ALLOCA]]
-// LLVM: ret ptr %[[RET]]
+// LLVMCIR-LABEL: define{{.*}} ptr @_Z19test_ref_cond_throwbRi(
+// LLVMCIR: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVMCIR: [[TRUE_BB]]:
+// LLVMCIR: %[[X_PTR:.*]] = load ptr, ptr %{{.*}}
+// LLVMCIR: br label %[[PHI_BB:.*]]
+// LLVMCIR: [[FALSE_BB]]:
+// LLVMCIR: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
+// LLVMCIR: unreachable
+// LLVMCIR: [[PHI_BB]]:
+// LLVMCIR: %[[PHI:.*]] = phi ptr [ %[[X_PTR]], %[[TRUE_BB]] ]
+// LLVMCIR: store ptr %[[PHI]], ptr %[[RET_ALLOCA:.*]], align 8
+// LLVMCIR: %[[RET:.*]] = load ptr, ptr %[[RET_ALLOCA]]
+// LLVMCIR: ret ptr %[[RET]]
// OGCG-LABEL: define{{.*}} ptr @_Z19test_ref_cond_throwbRi(
// OGCG: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
@@ -804,18 +795,18 @@ void test_assign_through_cond(bool c, int &x) {
// CIR-NEXT: }) : (!cir.bool) -> !cir.ptr<!s32i>
// CIR: cir.store{{.*}} %[[FIVE]], %[[RES]] : !s32i, !cir.ptr<!s32i>
-// LLVM-LABEL: define{{.*}} void @_Z24test_assign_through_condbRi(
-// LLVM: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// LLVM: [[TRUE_BB]]:
-// LLVM: %[[X_PTR:.*]] = load ptr, ptr %{{.*}}
-// LLVM: br label %[[PHI_BB:.*]]
-// LLVM: [[FALSE_BB]]:
-// LLVM: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
-// LLVM: unreachable
-// LLVM: [[PHI_BB]]:
-// LLVM: %[[PHI:.*]] = phi ptr [ %[[X_PTR]], %[[TRUE_BB]] ]
-// LLVM: store i32 5, ptr %[[PHI]], align 4
-// LLVM: ret void
+// LLVMCIR-LABEL: define{{.*}} void @_Z24test_assign_through_condbRi(
+// LLVMCIR: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVMCIR: [[TRUE_BB]]:
+// LLVMCIR: %[[X_PTR:.*]] = load ptr, ptr %{{.*}}
+// LLVMCIR: br label %[[PHI_BB:.*]]
+// LLVMCIR: [[FALSE_BB]]:
+// LLVMCIR: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
+// LLVMCIR: unreachable
+// LLVMCIR: [[PHI_BB]]:
+// LLVMCIR: %[[PHI:.*]] = phi ptr [ %[[X_PTR]], %[[TRUE_BB]] ]
+// LLVMCIR: store i32 5, ptr %[[PHI]], align 4
+// LLVMCIR: ret void
// OGCG-LABEL: define{{.*}} void @_Z24test_assign_through_condbRi(
// OGCG: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
@@ -846,17 +837,17 @@ int &test_member_cond_throw(bool c, struct s6 *p) {
// CIR-NEXT: }) : (!cir.bool) -> !cir.ptr<!s32i>
// CIR: cir.store %[[RES]], %{{.*}} : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
-// LLVM-LABEL: define{{.*}} ptr @_Z22test_member_cond_throwbP2s6(
-// LLVM: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// LLVM: [[TRUE_BB]]:
-// LLVM: %[[F0_PTR:.*]] = getelementptr{{.*}}%struct.s6, ptr %{{.*}}
-// LLVM: br label %[[PHI_BB:.*]]
-// LLVM: [[FALSE_BB]]:
-// LLVM: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
-// LLVM: unreachable
-// LLVM: [[PHI_BB]]:
-// LLVM: %[[PHI:.*]] = phi ptr [ %[[F0_PTR]], %[[TRUE_BB]] ]
-// LLVM: store ptr %[[PHI]], ptr %{{.*}}, align 8
+// LLVMCIR-LABEL: define{{.*}} ptr @_Z22test_member_cond_throwbP2s6(
+// LLVMCIR: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVMCIR: [[TRUE_BB]]:
+// LLVMCIR: %[[F0_PTR:.*]] = getelementptr{{.*}}%struct.s6, ptr %{{.*}}
+// LLVMCIR: br label %[[PHI_BB:.*]]
+// LLVMCIR: [[FALSE_BB]]:
+// LLVMCIR: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
+// LLVMCIR: unreachable
+// LLVMCIR: [[PHI_BB]]:
+// LLVMCIR: %[[PHI:.*]] = phi ptr [ %[[F0_PTR]], %[[TRUE_BB]] ]
+// LLVMCIR: store ptr %[[PHI]], ptr %{{.*}}, align 8
// OGCG-LABEL: define{{.*}} ptr @_Z22test_member_cond_throwbP2s6(
// OGCG: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
@@ -915,19 +906,6 @@ void test_agg_init_normal(bool c) {
// LLVM: [[END]]:
// LLVM: ret void
-// OGCG-LABEL: define{{.*}} void @_Z20test_agg_init_normalb(
-// OGCG: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// OGCG: [[TRUE_BB]]:
-// OGCG: store i32 1, ptr %{{.*}}
-// OGCG: store i32 2, ptr %{{.*}}
-// OGCG: br label %[[END:.*]]
-// OGCG: [[FALSE_BB]]:
-// OGCG: store i32 3, ptr %{{.*}}
-// OGCG: store i32 4, ptr %{{.*}}
-// OGCG: br label %[[END]]
-// OGCG: [[END]]:
-// OGCG: ret void
-
// Assignment context: the conditional materializes into a temporary that is
// then assigned to the target.
void test_agg_assign_throw(bool c, Agg &a) {
@@ -950,18 +928,18 @@ void test_agg_assign_throw(bool c, Agg &a) {
// CIR: %[[A_VAL:.*]] = cir.load %[[A_REF]]
// CIR: cir.call @_ZN3AggaSEOS_(%[[A_VAL]], %[[TMP]])
-// LLVM-LABEL: define{{.*}} void @_Z21test_agg_assign_throwbR3Agg(
-// LLVM: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// LLVM: [[TRUE_BB]]:
-// LLVM: call{{.*}} ptr @__cxa_allocate_exception
-// LLVM: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
-// LLVM: unreachable
-// LLVM: [[FALSE_BB]]:
-// LLVM: store i32 1, ptr %{{.*}}
-// LLVM: store i32 2, ptr %{{.*}}
-// LLVM: br label %[[END:.*]]
-// LLVM: [[END]]:
-// LLVM: call{{.*}} ptr @_ZN3AggaSEOS_(
+// LLVMCIR-LABEL: define{{.*}} void @_Z21test_agg_assign_throwbR3Agg(
+// LLVMCIR: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
+// LLVMCIR: [[TRUE_BB]]:
+// LLVMCIR: call{{.*}} ptr @__cxa_allocate_exception
+// LLVMCIR: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
+// LLVMCIR: unreachable
+// LLVMCIR: [[FALSE_BB]]:
+// LLVMCIR: store i32 1, ptr %{{.*}}
+// LLVMCIR: store i32 2, ptr %{{.*}}
+// LLVMCIR: br label %[[END:.*]]
+// LLVMCIR: [[END]]:
+// LLVMCIR: call{{.*}} ptr @_ZN3AggaSEOS_(
// OGCG-LABEL: define{{.*}} void @_Z21test_agg_assign_throwbR3Agg(
// OGCG: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
@@ -1014,28 +992,13 @@ void test_agg_nested_throw(bool c1, bool c2) {
// LLVM: store i32 3, ptr %{{.*}}
// LLVM: store i32 4, ptr %{{.*}}
-// OGCG-LABEL: define{{.*}} void @_Z21test_agg_nested_throwbb(
-// OGCG: br i1 %{{.*}}, label %[[OUTER_TRUE:.*]], label %[[OUTER_FALSE:.*]]
-// OGCG: [[OUTER_TRUE]]:
-// OGCG: br i1 %{{.*}}, label %[[INNER_TRUE:.*]], label %[[INNER_FALSE:.*]]
-// OGCG: [[INNER_TRUE]]:
-// OGCG: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
-// OGCG: unreachable
-// OGCG: [[INNER_FALSE]]:
-// OGCG: store i32 1, ptr %{{.*}}
-// OGCG: store i32 2, ptr %{{.*}}
-// OGCG: [[OUTER_FALSE]]:
-// OGCG: store i32 3, ptr %{{.*}}
-// OGCG: store i32 4, ptr %{{.*}}
-
// Call-argument context: the conditional materializes the argument temporary.
void test_agg_arg_throw(bool c) {
take(c ? throw 0 : Agg{1, 2});
}
// CIR-LABEL: cir.func{{.*}} @_Z18test_agg_arg_throwb(
-// CIR: %[[COERCE:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!rec_Agg>
-// CIR: %[[TMP:.*]] = cir.alloca "agg.tmp0" {{.*}} : !cir.ptr<!rec_Agg>
+// CIR: %[[TMP:.*]] = cir.alloca "agg.tmp0" align(4) : !cir.ptr<!rec_Agg>
// CIR: cir.if %{{.*}} {
// CIR: %[[EXC:.*]] = cir.alloc.exception{{.*}} -> !cir.ptr<!s32i>
// CIR: cir.throw %[[EXC]] : !cir.ptr<!s32i>, @_ZTIi
@@ -1044,10 +1007,8 @@ void test_agg_arg_throw(bool c) {
// CIR: cir.get_member %[[TMP]][0] {name = "x"}
// CIR: cir.get_member %[[TMP]][1] {name = "y"}
// CIR: }
-// CIR: %[[AGG:.*]] = cir.load{{.*}} %[[TMP]] : !cir.ptr<!rec_Agg>, !rec_Agg
-// CIR: cir.store %[[AGG]], %[[COERCE]] : !rec_Agg, !cir.ptr<!rec_Agg>
-// CIR: %[[COERCE_PTR:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!rec_Agg> -> !cir.ptr<!u64i>
-// CIR: %[[ARG:.*]] = cir.load %[[COERCE_PTR]] : !cir.ptr<!u64i>, !u64i
+// CIR: %[[TMP_PTR:.*]] = cir.cast bitcast %[[TMP]] : !cir.ptr<!rec_Agg> -> !cir.ptr<!u64i>
+// CIR: %[[ARG:.*]] = cir.load align(4) %[[TMP_PTR]] : !cir.ptr<!u64i>, !u64i
// CIR: cir.call @_Z4take3Agg(%[[ARG]]) : (!u64i) -> ()
// LLVM-LABEL: define{{.*}} void @_Z18test_agg_arg_throwb(
@@ -1061,15 +1022,3 @@ void test_agg_arg_throw(bool c) {
// LLVM: br label %[[END:.*]]
// LLVM: [[END]]:
// LLVM: call void @_Z4take3Agg(
-
-// OGCG-LABEL: define{{.*}} void @_Z18test_agg_arg_throwb(
-// OGCG: br i1 %{{.*}}, label %[[TRUE_BB:.*]], label %[[FALSE_BB:.*]]
-// OGCG: [[TRUE_BB]]:
-// OGCG: call void @__cxa_throw(ptr %{{.*}}, ptr @_ZTIi
-// OGCG: unreachable
-// OGCG: [[FALSE_BB]]:
-// OGCG: store i32 1, ptr %{{.*}}
-// OGCG: store i32 2, ptr %{{.*}}
-// OGCG: br label %[[END:.*]]
-// OGCG: [[END]]:
-// OGCG: call void @_Z4take3Agg(
diff --git a/clang/test/CIR/CodeGen/three-way-cmp.cpp b/clang/test/CIR/CodeGen/three-way-cmp.cpp
index c522e66d9edc77..0f133e6b5937af 100644
--- a/clang/test/CIR/CodeGen/three-way-cmp.cpp
+++ b/clang/test/CIR/CodeGen/three-way-cmp.cpp
@@ -196,15 +196,13 @@ void use_pseudo_ordering(HasMember m1, HasMember m2) {
// LLVM: store %struct.HasMember poison, ptr %[[M2_ALLOCA]]
// LLVM: %[[CALL_RES:.*]] = call i8 @_ZNK9HasMemberssERKS_(ptr {{.*}}%[[M1_ALLOCA]], ptr {{.*}}%[[M2_ALLOCA]])
// LLVM: store i8 %[[CALL_RES]], ptr %[[RETVAL]]
- // LLVM: %[[RETVAL_LOAD:.*]] = load %"class.std::__1::strong_ordering", ptr %[[RETVAL]]
- // LLVM: store %"class.std::__1::strong_ordering" %[[RETVAL_LOAD]], ptr %[[G_ALLOCA]]
+ // LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 1 %[[G_ALLOCA]], ptr align 1 %[[RETVAL]], i64 1, i1 false)
// LLVM: ret void
// LLVM: }
// LLVM: define {{.*}} i8 @_ZNK9HasMemberssERKS_(ptr {{.*}}, ptr {{.*}})
// LLVM: %[[CMP_COPY:.*]] = alloca %"class.std::__1::strong_ordering"
// LLVM: %[[CMP_TEMP:.*]] = alloca %"class.std::__1::strong_ordering"
- // LLVM: %[[VALUE_SLOT:.*]] = alloca %"class.std::__1::strong_ordering"
// LLVM: %[[LHS_ALLOCA:.*]] = alloca ptr
// LLVM: %[[RHS_ALLOCA:.*]] = alloca ptr
// LLVM: %[[RESULT_SLOT:.*]] = alloca %"class.std::__1::strong_ordering"
@@ -243,10 +241,8 @@ void use_pseudo_ordering(HasMember m1, HasMember m2) {
// LLVM: [[COPY_BLOCK]]:
// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 1 %[[CMP_COPY]], ptr align 1 %[[CMP_RES]], i64 1, i1 false)
// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 1 %[[CMP_TEMP]], ptr align 1 %[[CMP_COPY]], i64 1, i1 false)
- // LLVM: %[[CMP_TEMP_LOAD:.*]] = load %"class.std::__1::strong_ordering", ptr %[[CMP_TEMP]]
- // LLVM: store %"class.std::__1::strong_ordering" %[[CMP_TEMP_LOAD]], ptr %[[VALUE_SLOT]]
- // LLVM: %[[CMP_I8:.*]] = load i8, ptr %[[VALUE_SLOT]]
- // LLVM: %[[SO_NE_RES:.*]] = call noundef zeroext i1 @_ZNSt3__1neENS_15strong_orderingEMNS_19_CmpUnspecifiedTypeEFvvE(i8 %[[CMP_I8]], i64 0, i64 0)
+ // LLVM-NEXT: %[[CMP_I8:.*]] = load i8, ptr %[[CMP_TEMP]], align 1
+ // LLVM-NEXT: %[[SO_NE_RES:.*]] = call noundef zeroext i1 @_ZNSt3__1neENS_15strong_orderingEMNS_19_CmpUnspecifiedTypeEFvvE(i8 %[[CMP_I8]], i64 0, i64 0)
// LLVM: br i1 %[[SO_NE_RES]], label %[[SO_NE_RES_TRUE:.*]], label %[[SO_NE_RES_FALSE:.*]]
//
// LLVM: [[SO_NE_RES_TRUE]]:
diff --git a/clang/test/CIR/CodeGen/union.c b/clang/test/CIR/CodeGen/union.c
index a655bcce9a8f72..ebb3e369e884c6 100644
--- a/clang/test/CIR/CodeGen/union.c
+++ b/clang/test/CIR/CodeGen/union.c
@@ -116,9 +116,8 @@ void shouldGenerateUnionAccess(union U2 u) {
// CIR-NEXT: %[[COERCE:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
// CIR-NEXT: cir.store{{.*}} %[[ARG]], %[[COERCE]] : !u64i, !cir.ptr<!u64i>
// CIR-NEXT: %[[CAST:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_U2>
-// CIR-NEXT: %[[ARG_VAL:.*]] = cir.load{{.*}} %[[CAST]] : !cir.ptr<!rec_U2>, !rec_U2
// CIR-NEXT: %[[U:.*]] = cir.alloca "u" align(8) init : !cir.ptr<!rec_U2>
-// CIR-NEXT: cir.store{{.*}} %[[ARG_VAL]], %[[U]] : !rec_U2, !cir.ptr<!rec_U2>
+// CIR-NEXT: cir.copy %[[CAST]] align(8) to %[[U]] align(8) : !cir.ptr<!rec_U2>
// CIR-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0> : !s8i
// CIR-NEXT: %[[B_PTR:.*]] = cir.get_member %[[U]][0] {name = "b"} : !cir.ptr<!rec_U2> -> !cir.ptr<!s8i>
// CIR-NEXT: cir.store{{.*}} %[[ZERO]], %[[B_PTR]] : !s8i, !cir.ptr<!s8i>
@@ -144,9 +143,8 @@ void shouldGenerateUnionAccess(union U2 u) {
// LLVM: define{{.*}} void @shouldGenerateUnionAccess(i64 %[[ARG:.*]])
// LLVM-NEXT: %[[COERCE:.*]] = alloca i64, align 8
// LLVM-NEXT: store i64 %[[ARG]], ptr %[[COERCE]], align 8
-// LLVM-NEXT: %[[ARG_VAL:.*]] = load %union.U2, ptr %[[COERCE]], align 8
// LLVM-NEXT: %[[U:.*]] = alloca %union.U2, align 8
-// LLVM-NEXT: store %union.U2 %[[ARG_VAL]], ptr %[[U]], align 8
+// LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[U]], ptr align 8 %[[COERCE]], i64 8, i1 false)
// LLVM-NEXT: store i8 0, ptr %[[U]], align 8
// LLVM-NEXT: %[[B_VAL:.*]] = load i8, ptr %[[U]], align 8
// LLVM-NEXT: store i32 1, ptr %[[U]], align 8
@@ -180,9 +178,8 @@ void f3(union U3 u) {
// CIR-NEXT: %[[COERCE:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!cir.int<u, 40>>
// CIR-NEXT: cir.store{{.*}} %[[ARG]], %[[COERCE]] : !cir.int<u, 40>, !cir.ptr<!cir.int<u, 40>>
// CIR-NEXT: %[[CAST:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!cir.int<u, 40>> -> !cir.ptr<!rec_U3>
-// CIR-NEXT: %[[ARG_VAL:.*]] = cir.load{{.*}} %[[CAST]] : !cir.ptr<!rec_U3>, !rec_U3
// CIR-NEXT: %[[U:.*]] = cir.alloca "u" align(1) init : !cir.ptr<!rec_U3>
-// CIR-NEXT: cir.store{{.*}} %[[ARG_VAL]], %[[U]] : !rec_U3, !cir.ptr<!rec_U3>
+// CIR-NEXT: cir.copy %[[CAST]] align(8) to %[[U]] align(1) : !cir.ptr<!rec_U3>
// CIR-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0> : !s8i
// CIR-NEXT: %[[C_PTR:.*]] = cir.get_member %[[U]][0] {name = "c"} : !cir.ptr<!rec_U3> -> !cir.ptr<!cir.array<!s8i x 5>>
// CIR-NEXT: %[[IDX:.*]] = cir.const #cir.int<2> : !s64i
@@ -193,9 +190,8 @@ void f3(union U3 u) {
// LLVM: define{{.*}} void @f3(i40 %[[ARG:.*]])
// LLVM-NEXT: %[[COERCE:.*]] = alloca i40, align 8
// LLVM-NEXT: store i40 %[[ARG]], ptr %[[COERCE]], align 8
-// LLVM-NEXT: %[[ARG_VAL:.*]] = load %union.U3, ptr %[[COERCE]], align 1
// LLVM-NEXT: %[[U:.*]] = alloca %union.U3, align 1
-// LLVM-NEXT: store %union.U3 %[[ARG_VAL]], ptr %[[U]], align 1
+// LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 %[[U]], ptr align 8 %[[COERCE]], i64 5, i1 false)
// LLVM-NEXT: %[[ELEM_PTR:.*]] = getelementptr [5 x i8], ptr %[[U]], i32 0, i64 2
// LLVM-NEXT: store i8 0, ptr %[[ELEM_PTR]], align 1
// LLVM-NEXT: ret void
@@ -216,9 +212,8 @@ void f5(union U4 u) {
// CIR-NEXT: %[[COERCE:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
// CIR-NEXT: cir.store{{.*}} %[[ARG]], %[[COERCE]] : !u64i, !cir.ptr<!u64i>
// CIR-NEXT: %[[CAST:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_U4>
-// CIR-NEXT: %[[ARG_VAL:.*]] = cir.load{{.*}} %[[CAST]] : !cir.ptr<!rec_U4>, !rec_U4
// CIR-NEXT: %[[U:.*]] = cir.alloca "u" align(4) init : !cir.ptr<!rec_U4>
-// CIR-NEXT: cir.store{{.*}} %[[ARG_VAL]], %[[U]] : !rec_U4, !cir.ptr<!rec_U4>
+// CIR-NEXT: cir.copy %[[CAST]] align(8) to %[[U]] align(4) : !cir.ptr<!rec_U4>
// CIR-NEXT: %[[CHAR_VAL:.*]] = cir.const #cir.int<65> : !s8i
// CIR-NEXT: %[[C_PTR:.*]] = cir.get_member %[[U]][0] {name = "c"} : !cir.ptr<!rec_U4> -> !cir.ptr<!cir.array<!s8i x 5>>
// CIR-NEXT: %[[IDX:.*]] = cir.const #cir.int<4> : !s64i
@@ -229,9 +224,8 @@ void f5(union U4 u) {
// LLVM: define{{.*}} void @f5(i64 %[[ARG:.*]])
// LLVM-NEXT: %[[COERCE:.*]] = alloca i64, align 8
// LLVM-NEXT: store i64 %[[ARG]], ptr %[[COERCE]], align 8
-// LLVM-NEXT: %[[ARG_VAL:.*]] = load %union.U4, ptr %[[COERCE]], align 4
// LLVM-NEXT: %[[U:.*]] = alloca %union.U4, align 4
-// LLVM-NEXT: store %union.U4 %[[ARG_VAL]], ptr %[[U]], align 4
+// LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[U]], ptr align 8 %[[COERCE]], i64 8, i1 false)
// LLVM-NEXT: %[[ELEM_PTR:.*]] = getelementptr [5 x i8], ptr %[[U]], i32 0, i64 4
// LLVM-NEXT: store i8 65, ptr %[[ELEM_PTR]], align 4
// LLVM-NEXT: ret void
diff --git a/clang/test/CIR/CodeGen/union.cpp b/clang/test/CIR/CodeGen/union.cpp
index cd60ba003b6a8c..eb3b0cd4d694de 100644
--- a/clang/test/CIR/CodeGen/union.cpp
+++ b/clang/test/CIR/CodeGen/union.cpp
@@ -42,9 +42,11 @@ void shouldGenerateUnionAccess(union U u) {
// CIR: cir.store{{.*}} %{{.+}}, %[[#BASE]] : !cir.double, !cir.ptr<!cir.double>
// CIR: %[[#BASE:]] = cir.get_member %[[U]][4] {name = "d"} : !cir.ptr<!rec_U> -> !cir.ptr<!cir.double>
-// LLVM: define {{.*}}shouldGenerateUnionAccess1U(i64 %{{[^,)]+}})
-// LLVM: %[[BASE:.*]] = alloca %union.U
-// LLVM: store %union.U %{{.*}}, ptr %[[BASE]]
+// LLVM: define {{.*}}shouldGenerateUnionAccess1U(i64 %[[ARG:[^,)]+]])
+// LLVM: %[[SLOT:.*]] = alloca i64, align 8
+// LLVM: store i64 %[[ARG]], ptr %[[SLOT]], align 8
+// LLVM: %[[BASE:.*]] = alloca %union.U, align 8
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[BASE]], ptr align 8 %[[SLOT]], i64 8, i1 false)
// LLVM: store i8 1, ptr %[[BASE]]
// LLVM: store i32 1, ptr %[[BASE]]
// LLVM: store float 1.000000e-01, ptr %[[BASE]]
diff --git a/clang/test/CIR/CodeGen/var-arg-aggregate.c b/clang/test/CIR/CodeGen/var-arg-aggregate.c
index 3823e866c75db8..4d1115be345c6e 100644
--- a/clang/test/CIR/CodeGen/var-arg-aggregate.c
+++ b/clang/test/CIR/CodeGen/var-arg-aggregate.c
@@ -111,8 +111,7 @@ struct Big varargs_aggregate_memory(int count, ...) {
// CIR: cir.yield %[[OVERFLOW_B]] : !cir.ptr<!u8i>
// CIR: }) : (!cir.bool) -> !cir.ptr<!u8i>
// CIR: %[[VA_ARG_B:.+]] = cir.cast bitcast %[[VA_ARG]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_Bar>
-// CIR: %[[VA_ARG_V:.+]] = cir.load align(4) %[[VA_ARG_B]] : !cir.ptr<!rec_Bar>, !rec_Bar
-// CIR: cir.store{{.*}} %[[VA_ARG_V]], %[[TMP_ADDR]] : !rec_Bar, !cir.ptr<!rec_Bar>
+// CIR: cir.copy %[[VA_ARG_B]] align(4) to %[[TMP_ADDR]] align(4) : !cir.ptr<!rec_Bar>
// CIR: cir.copy %[[TMP_ADDR]] align(4) to %[[RET_ADDR]] align(4) : !cir.ptr<!rec_Bar>
// CIR: %[[VA_PTR2:.+]] = cir.cast array_to_ptrdecay %[[VAAREA]] : !cir.ptr<!cir.array<!rec___va_list_tag x 1>> -> !cir.ptr<!rec___va_list_tag>
// CIR: cir.va_end %[[VA_PTR2]] : !cir.ptr<!rec___va_list_tag>
@@ -143,7 +142,7 @@ struct Big varargs_aggregate_memory(int count, ...) {
// CIR: cir.yield %[[OVERFLOW_B]] : !cir.ptr<!u8i>
// CIR: }) : (!cir.bool) -> !cir.ptr<!u8i>
// CIR: %[[VA_ARG_B:.+]] = cir.cast bitcast %[[VA_ARG]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_LongPair>
-// CIR: %[[VA_ARG_V:.+]] = cir.load align(8) %[[VA_ARG_B]] : !cir.ptr<!rec_LongPair>, !rec_LongPair
+// CIR: cir.copy %[[VA_ARG_B]] align(8) to %{{.+}} align(8) : !cir.ptr<!rec_LongPair>
// SSE pair: both eightbytes SSE, 16 bytes apart, reassembled via a temp.
// CIR-LABEL: cir.func {{.*}} @varargs_aggregate_sse_pair(
@@ -177,7 +176,7 @@ struct Big varargs_aggregate_memory(int count, ...) {
// CIR: cir.yield %{{.+}} : !cir.ptr<!u8i>
// CIR: }) : (!cir.bool) -> !cir.ptr<!u8i>
// CIR: %[[VA_ARG_B:.+]] = cir.cast bitcast %[[VA_ARG]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_DoublePair>
-// CIR: %[[VA_ARG_V:.+]] = cir.load align(8) %[[VA_ARG_B]] : !cir.ptr<!rec_DoublePair>, !rec_DoublePair
+// CIR: cir.copy %[[VA_ARG_B]] align(8) to %{{.+}} align(8) : !cir.ptr<!rec_DoublePair>
// MEMORY class: three eightbytes never fit in registers.
// CIR-LABEL: cir.func {{.*}} @varargs_aggregate_memory(
@@ -188,7 +187,7 @@ struct Big varargs_aggregate_memory(int count, ...) {
// CIR: %[[OVERFLOW_NEXT:.+]] = cir.ptr_stride %[[OVERFLOW_B]], %[[STRIDE]] : (!cir.ptr<!u8i>, !s32i) -> !cir.ptr<!u8i>
// CIR: cir.store %[[OVERFLOW_NEXT]], %{{.+}} : !cir.ptr<!u8i>, !cir.ptr<!cir.ptr<!u8i>>
// CIR: %[[VA_ARG_B:.+]] = cir.cast bitcast %[[OVERFLOW_B]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_Big>
-// CIR: %[[VA_ARG_V:.+]] = cir.load align(8) %[[VA_ARG_B]] : !cir.ptr<!rec_Big>, !rec_Big
+// CIR: cir.copy %[[VA_ARG_B]] align(8) to %{{.+}} align(8) : !cir.ptr<!rec_Big>
// LLVM-LABEL: define dso_local { <2 x float>, i32 } @varargs_aggregate_mixed_pair(i32 noundef %{{.*}}, ...)
// OGCG: %[[RETVAL:.+]] = alloca %struct.Bar, align 4
@@ -227,13 +226,10 @@ struct Big varargs_aggregate_memory(int count, ...) {
// LLVM: store ptr %[[OVERFLOW_NEXT]], ptr %[[OVERFLOW_P]], align 8
// LLVM: br label %[[END_BB]]
// LLVM: [[END_BB]]:
-// The fetched value is materialized by loading the record on one side and by
-// copying it on the other, so only the address feeding it is shared.
// LLVMCIR: %[[ADDR:.+]] = phi ptr [ %[[OVERFLOW]], %[[MEM_BB]] ], [ %[[REG_TMP]], %[[REG_BB]] ]
-// LLVMCIR: %[[VA_ARG:.+]] = load %struct.Bar, ptr %[[ADDR]], align 4
-// LLVMCIR: store %struct.Bar %[[VA_ARG]], ptr %{{.*}}, align 4
+// LLVMCIR: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %{{.+}}, ptr align 4 %[[ADDR]], i64 12, i1 false)
// OGCG: %[[ADDR:.+]] = phi ptr [ %{{.*}}, %[[REG_BB]] ], [ %{{.*}}, %[[MEM_BB]] ]
-// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[RETVAL_ADDR:.+]], ptr align 4 %{{.*}}, i64 12, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[RETVAL_ADDR:.+]], ptr align 4 %{{[^,]+}}, i64 12, i1 false)
// LLVM: call void @llvm.va_end.p0(ptr %{{.+}})
// LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[COERCE:.+]], ptr align 4 %[[RETVAL_ADDR]], i64 12, i1 false)
// LLVM-NEXT: %[[RES:.+]] = load { <2 x float>, i32 }, ptr %[[COERCE]], align 8
@@ -261,10 +257,8 @@ struct Big varargs_aggregate_memory(int count, ...) {
// LLVM: br label %[[END_BB]]
// LLVM: [[END_BB]]:
// LLVMCIR: %[[ADDR:.+]] = phi ptr [ %[[OVERFLOW]], %[[MEM_BB]] ], [ %[[REG_ADDR]], %[[REG_BB]] ]
-// LLVMCIR: %[[VA_ARG:.+]] = load %struct.LongPair, ptr %[[ADDR]], align 8
-// LLVMCIR: store %struct.LongPair %[[VA_ARG]], ptr %{{.*}}, align 8
// OGCG: %[[ADDR:.+]] = phi ptr [ %{{.*}}, %[[REG_BB]] ], [ %{{.*}}, %[[MEM_BB]] ]
-// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.*}}, ptr align 8 %[[ADDR]], i64 16, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.*}}, ptr align 8 %[[ADDR]], i64 16, i1 false)
// LLVM-LABEL: define dso_local { double, double } @varargs_aggregate_sse_pair(i32 noundef %{{.*}}, ...)
// LLVM: %[[FP_OFFSET_P:.+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %{{.*}}, i32 0, i32 1
@@ -290,19 +284,15 @@ struct Big varargs_aggregate_memory(int count, ...) {
// LLVM: br label %[[END_BB]]
// LLVM: [[END_BB]]:
// LLVMCIR: %[[ADDR:.+]] = phi ptr [ %{{.*}}, %[[MEM_BB]] ], [ %[[REG_TMP]], %[[REG_BB]] ]
-// LLVMCIR: %[[VA_ARG:.+]] = load %struct.DoublePair, ptr %[[ADDR]], align 8
-// LLVMCIR: store %struct.DoublePair %[[VA_ARG]], ptr %{{.*}}, align 8
// OGCG: %[[ADDR:.+]] = phi ptr [ %{{.*}}, %[[REG_BB]] ], [ %{{.*}}, %[[MEM_BB]] ]
-// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.*}}, ptr align 8 %[[ADDR]], i64 16, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.*}}, ptr align 8 %[[ADDR]], i64 16, i1 false)
// LLVM-LABEL: define dso_local void @varargs_aggregate_memory(ptr dead_on_unwind noalias writable sret(%struct.Big) align 8 %{{.*}}, i32 noundef %{{.*}}, ...)
// LLVM: %[[OVERFLOW_P:.+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %{{.*}}, i32 0, i32 2
// LLVM: %[[OVERFLOW:.+]] = load ptr, ptr %[[OVERFLOW_P]], align 8
// LLVM: %[[OVERFLOW_NEXT:.+]] = getelementptr i8, ptr %[[OVERFLOW]], i{{32|64}} 24
// LLVM: store ptr %[[OVERFLOW_NEXT]], ptr %[[OVERFLOW_P]], align 8
-// LLVMCIR: %[[VA_ARG:.+]] = load %struct.Big, ptr %[[OVERFLOW]], align 8
-// LLVMCIR: store %struct.Big %[[VA_ARG]], ptr %{{.*}}, align 8
-// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.*}}, ptr align 8 %{{.*}}, i64 24, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.*}}, ptr align 8 %[[OVERFLOW]], i64 24, i1 false)
struct RevMixed {
long a;
@@ -348,7 +338,7 @@ struct RevMixed varargs_aggregate_mixed_pair_rev(int count, ...) {
// CIR: cir.yield %[[REG_TMP_B]] : !cir.ptr<!u8i>
// CIR: }) : (!cir.bool) -> !cir.ptr<!u8i>
// CIR: %[[RESULT_P:.+]] = cir.cast bitcast %[[ADDR]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_RevMixed>
-// CIR: cir.load align(8) %[[RESULT_P]] : !cir.ptr<!rec_RevMixed>, !rec_RevMixed
+// CIR: cir.copy %[[RESULT_P]] align(8) to %{{.+}} align(8) : !cir.ptr<!rec_RevMixed>
// LLVM-LABEL: define dso_local { i64, double } @varargs_aggregate_mixed_pair_rev(i32 noundef %{{.*}}, ...)
// LLVMCIR: %[[GP_OFFSET:.+]] = load i32, ptr %{{.+}}, align 4
@@ -375,8 +365,7 @@ struct RevMixed varargs_aggregate_mixed_pair_rev(int count, ...) {
// OGCG: store double %[[HI_VAL]], ptr %{{.+}}, align 8
// LLVM: add i32 %[[GP_OFFSET]], 8
// LLVM: add i32 %[[FP_OFFSET]], 16
-// LLVMCIR: load %struct.RevMixed, ptr %{{.+}}, align 8
-// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.+}}, ptr align 8 %{{.+}}, i64 16, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.+}}, ptr align 8 %{{.+}}, i64 16, i1 false)
struct __attribute__((aligned(16))) OverAligned {
long a;
@@ -403,8 +392,7 @@ struct OverAligned varargs_aggregate_overaligned(int count, ...) {
// CIR: %[[ADDR:.+]] = cir.ternary(%[[FITS]], true {
// CIR: %[[REG_ADDR:.+]] = cir.ptr_stride %{{.+}}, %[[GP_OFFSET]] : (!cir.ptr<!u8i>, !u32i) -> !cir.ptr<!u8i>
// CIR: %[[REG_ADDR_V:.+]] = cir.cast bitcast %[[REG_ADDR]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_OverAligned>
-// CIR: %[[VAL:.+]] = cir.load align(8) %[[REG_ADDR_V]] : !cir.ptr<!rec_OverAligned>, !rec_OverAligned
-// CIR: cir.store %[[VAL]], %[[TEMP]] : !rec_OverAligned, !cir.ptr<!rec_OverAligned>
+// CIR: cir.copy %[[REG_ADDR_V]] align(8) to %[[TEMP]] align(16) : !cir.ptr<!rec_OverAligned>
// CIR: %[[TEMP_B:.+]] = cir.cast bitcast %[[TEMP]] : !cir.ptr<!rec_OverAligned> -> !cir.ptr<!u8i>
// CIR: cir.yield %[[TEMP_B]] : !cir.ptr<!u8i>
// CIR: }, false {
@@ -415,25 +403,23 @@ struct OverAligned varargs_aggregate_overaligned(int count, ...) {
// CIR: cir.yield %[[ALIGNED]] : !cir.ptr<!u8i>
// CIR: }) : (!cir.bool) -> !cir.ptr<!u8i>
// CIR: %[[RESULT_P:.+]] = cir.cast bitcast %[[ADDR]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_OverAligned>
-// CIR: cir.load align(16) %[[RESULT_P]] : !cir.ptr<!rec_OverAligned>, !rec_OverAligned
+// CIR: cir.copy %[[RESULT_P]] align(16) to %{{.+}} align(16) : !cir.ptr<!rec_OverAligned>
// LLVM-LABEL: define dso_local { i64, i64 } @varargs_aggregate_overaligned(i32 noundef %{{.*}}, ...)
// LLVMCIR: %[[GP_OFFSET:.+]] = load i32, ptr %{{.+}}, align 4
// OGCG: %[[GP_OFFSET:.+]] = load i32, ptr %{{.+}}, align 16
// LLVM: icmp ule i32 %[[GP_OFFSET]], 32
-// The register slot is only 8-aligned, so the copy reads at 8 and stores
-// into a temp carrying the record's declared 16. The temp is bound at its
-// alloca, so that an under-aligned one cannot satisfy the store below.
+// The register slot is only 8-aligned, so the copy reads at 8 and writes a
+// temp carrying the record's declared 16. The temp is bound at its alloca, so
+// that an under-aligned one cannot satisfy the copy below.
// LLVMCIR: %[[TEMP:.+]] = alloca %struct.OverAligned, align 16
-// LLVMCIR: %[[VAL:.+]] = load %struct.OverAligned, ptr %{{.+}}, align 8
-// LLVMCIR: store %struct.OverAligned %[[VAL]], ptr %[[TEMP]], align 8
+// LLVMCIR: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[TEMP]], ptr align 8 %{{.+}}, i64 16, i1 false)
// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[TEMP:.+]], ptr align 8 %{{.+}}, i64 16, i1 false)
// LLVM: %[[ALIGNED:.+]] = call ptr @llvm.ptrmask.p0.i64(ptr %{{.+}}, i64 -16)
// LLVM: %[[MEM_NEXT:.+]] = getelementptr i8, ptr %[[ALIGNED]], i{{32|64}} 16
// LLVMCIR: %[[ADDR:.+]] = phi ptr [ %[[ALIGNED]], %{{.+}} ], [ %[[TEMP]], %{{.+}} ]
// OGCG: %[[ADDR:.+]] = phi ptr [ %[[TEMP]], %{{.+}} ], [ %[[ALIGNED]], %{{.+}} ]
-// LLVMCIR: load %struct.OverAligned, ptr %[[ADDR]], align 16
-// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{.+}}, ptr align 16 %[[ADDR]], i64 16, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{.+}}, ptr align 16 %[[ADDR]], i64 16, i1 false)
// A record can require more alignment than the types of its members imply,
// and the attribute that raises it need not sit on the record: it can sit on a
@@ -457,12 +443,11 @@ long varargs_overaligned_union(int count, ...) {
// CIR-LABEL: cir.func {{.*}} @varargs_overaligned_union(
// CIR: %[[ALIGNED:.+]] = cir.ptr_mask %{{.+}}, %{{.+}} : (!cir.ptr<!u8i>, !s64i) -> !cir.ptr<!u8i>
// CIR: %[[RESULT_P:.+]] = cir.cast bitcast %{{.+}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_OverAlignedUnion>
-// CIR: cir.load align(32) %[[RESULT_P]] : !cir.ptr<!rec_OverAlignedUnion>, !rec_OverAlignedUnion
+// CIR: cir.copy %[[RESULT_P]] align(32) to %{{.+}} align(32) : !cir.ptr<!rec_OverAlignedUnion>
// LLVM-LABEL: define dso_local i64 @varargs_overaligned_union(i32 noundef %{{.*}}, ...)
// LLVM: %[[ALIGNED:.+]] = call ptr @llvm.ptrmask.p0.i64(ptr %{{.+}}, i64 -32)
-// LLVMCIR: load %union.OverAlignedUnion, ptr %{{.+}}, align 32
-// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 32 %{{.+}}, ptr align 32 %[[ALIGNED]], i64 32, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 32 %{{.+}}, ptr align 32 %[[ALIGNED]], i64 32, i1 false)
struct FieldOverAligned {
__attribute__((aligned(32))) char c;
@@ -480,12 +465,11 @@ long varargs_field_overaligned(int count, ...) {
// CIR-LABEL: cir.func {{.*}} @varargs_field_overaligned(
// CIR: %[[ALIGNED:.+]] = cir.ptr_mask %{{.+}}, %{{.+}} : (!cir.ptr<!u8i>, !s64i) -> !cir.ptr<!u8i>
// CIR: %[[RESULT_P:.+]] = cir.cast bitcast %{{.+}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_FieldOverAligned>
-// CIR: cir.load align(32) %[[RESULT_P]] : !cir.ptr<!rec_FieldOverAligned>, !rec_FieldOverAligned
+// CIR: cir.copy %[[RESULT_P]] align(32) to %{{.+}} align(32) : !cir.ptr<!rec_FieldOverAligned>
// LLVM-LABEL: define dso_local i64 @varargs_field_overaligned(i32 noundef %{{.*}}, ...)
// LLVM: %[[ALIGNED:.+]] = call ptr @llvm.ptrmask.p0.i64(ptr %{{.+}}, i64 -32)
-// LLVMCIR: load %struct.FieldOverAligned, ptr %{{.+}}, align 32
-// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 32 %{{.+}}, ptr align 32 %[[ALIGNED]], i64 32, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 32 %{{.+}}, ptr align 32 %[[ALIGNED]], i64 32, i1 false)
struct HoldsOverAlignedUnion {
union OverAlignedUnion u;
@@ -503,12 +487,11 @@ long varargs_holds_overaligned_union(int count, ...) {
// CIR-LABEL: cir.func {{.*}} @varargs_holds_overaligned_union(
// CIR: %[[ALIGNED:.+]] = cir.ptr_mask %{{.+}}, %{{.+}} : (!cir.ptr<!u8i>, !s64i) -> !cir.ptr<!u8i>
// CIR: %[[RESULT_P:.+]] = cir.cast bitcast %{{.+}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_HoldsOverAlignedUnion>
-// CIR: cir.load align(32) %[[RESULT_P]] : !cir.ptr<!rec_HoldsOverAlignedUnion>, !rec_HoldsOverAlignedUnion
+// CIR: cir.copy %[[RESULT_P]] align(32) to %{{.+}} align(32) : !cir.ptr<!rec_HoldsOverAlignedUnion>
// LLVM-LABEL: define dso_local i64 @varargs_holds_overaligned_union(i32 noundef %{{.*}}, ...)
// LLVM: %[[ALIGNED:.+]] = call ptr @llvm.ptrmask.p0.i64(ptr %{{.+}}, i64 -32)
-// LLVMCIR: load %struct.HoldsOverAlignedUnion, ptr %{{.+}}, align 32
-// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 32 %{{.+}}, ptr align 32 %[[ALIGNED]], i64 32, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 32 %{{.+}}, ptr align 32 %[[ALIGNED]], i64 32, i1 false)
// Packing lowers the alignment the members imply, and an attribute can still
// raise the record above it.
@@ -529,9 +512,8 @@ long varargs_packed_overaligned(int count, ...) {
// CIR-LABEL: cir.func {{.*}} @varargs_packed_overaligned(
// CIR: %[[ALIGNED:.+]] = cir.ptr_mask %{{.+}}, %{{.+}} : (!cir.ptr<!u8i>, !s64i) -> !cir.ptr<!u8i>
// CIR: %[[RESULT_P:.+]] = cir.cast bitcast %{{.+}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_PackedOverAligned>
-// CIR: cir.load align(32) %[[RESULT_P]] : !cir.ptr<!rec_PackedOverAligned>, !rec_PackedOverAligned
+// CIR: cir.copy %[[RESULT_P]] align(32) to %{{.+}} align(32) : !cir.ptr<!rec_PackedOverAligned>
// LLVM-LABEL: define dso_local i64 @varargs_packed_overaligned(i32 noundef %{{.*}}, ...)
// LLVM: %[[ALIGNED:.+]] = call ptr @llvm.ptrmask.p0.i64(ptr %{{.+}}, i64 -32)
-// LLVMCIR: load %struct.PackedOverAligned, ptr %{{.+}}, align 32
-// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 32 %{{.+}}, ptr align 32 %[[ALIGNED]], i64 32, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 32 %{{.+}}, ptr align 32 %[[ALIGNED]], i64 32, i1 false)
diff --git a/clang/test/CIR/CodeGen/var-arg-direct-offset.cpp b/clang/test/CIR/CodeGen/var-arg-direct-offset.cpp
index 97c091f10ac776..0aa169b9fcf9e3 100644
--- a/clang/test/CIR/CodeGen/var-arg-direct-offset.cpp
+++ b/clang/test/CIR/CodeGen/var-arg-direct-offset.cpp
@@ -54,7 +54,7 @@ EmptyLow varargs_empty_low(int count, ...) {
// CIR: %[[YIELDED:.+]] = cir.cast bitcast %[[TEMP]] : !cir.ptr<!rec_EmptyLow> -> !cir.ptr<!u8i>
// CIR: cir.yield %[[YIELDED]] : !cir.ptr<!u8i>
// CIR: %[[RESULT_P:.+]] = cir.cast bitcast %[[ADDR]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_EmptyLow>
-// CIR: cir.load align(8) %[[RESULT_P]] : !cir.ptr<!rec_EmptyLow>, !rec_EmptyLow
+// CIR: cir.copy %[[RESULT_P]] align(8) to %{{.+}} align(8) : !cir.ptr<!rec_EmptyLow>
// LLVM-LABEL: define dso_local i64 @_Z17varargs_empty_lowiz(i32 noundef %{{.*}}, ...)
// LLVM: %[[GP_P:.+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %{{.+}}, i32 0, i32 0
@@ -82,8 +82,7 @@ EmptyLow varargs_empty_low(int count, ...) {
// LLVMCIR: %[[ADDR:.+]] = phi ptr [ %[[OVERFLOW]], %{{.+}} ], [ %[[TEMP]], %{{.+}} ]
// OGCG: %[[ADDR:.+]] = phi ptr [ %[[TEMP]], %{{.+}} ], [ %[[OVERFLOW]], %{{.+}} ]
-// LLVMCIR: load %struct.EmptyLow, ptr %[[ADDR]], align 8
-// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.+}}, ptr align 8 %[[ADDR]], i64 16, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.+}}, ptr align 8 %[[ADDR]], i64 16, i1 false)
double varargs_empty_low_sse(int count, ...) {
__builtin_va_list args;
@@ -118,7 +117,7 @@ double varargs_empty_low_sse(int count, ...) {
// CIR: cir.yield %[[YIELDED]] : !cir.ptr<!u8i>
// CIR: }) : (!cir.bool) -> !cir.ptr<!u8i>
// CIR: %[[RESULT_P:.+]] = cir.cast bitcast %[[ADDR]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_EmptyLowSse>
-// CIR: cir.load align(8) %[[RESULT_P]] : !cir.ptr<!rec_EmptyLowSse>, !rec_EmptyLowSse
+// CIR: cir.copy %[[RESULT_P]] align(8) to %{{.+}} align(8) : !cir.ptr<!rec_EmptyLowSse>
// LLVM-LABEL: define dso_local noundef double @_Z21varargs_empty_low_sseiz(i32 noundef %{{.*}}, ...)
// LLVM: %[[FP_P:.+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %{{.+}}, i32 0, i32 1
@@ -131,8 +130,7 @@ double varargs_empty_low_sse(int count, ...) {
// LLVM: store i32 %[[BUMPED]], ptr %[[FP_P]], align 4
// LLVMCIR: %[[ADDR:.+]] = phi ptr [ %{{.+}}, %{{.+}} ], [ %[[TEMP]], %{{.+}} ]
// OGCG: %[[ADDR:.+]] = phi ptr [ %[[TEMP]], %{{.+}} ], [ %{{.+}}, %{{.+}} ]
-// LLVMCIR: load %struct.EmptyLowSse, ptr %[[ADDR]], align 8
-// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.+}}, ptr align 8 %[[ADDR]], i64 16, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.+}}, ptr align 8 %[[ADDR]], i64 16, i1 false)
long varargs_tail_pad(int count, ...) {
__builtin_va_list args;
@@ -157,7 +155,7 @@ long varargs_tail_pad(int count, ...) {
// CIR: %[[YIELDED:.+]] = cir.cast bitcast %[[TEMP]] : !cir.ptr<!rec_TailPad> -> !cir.ptr<!u8i>
// CIR: cir.yield %[[YIELDED]] : !cir.ptr<!u8i>
// CIR: %[[RESULT_P:.+]] = cir.cast bitcast %[[ADDR]] : !cir.ptr<!u8i> -> !cir.ptr<!rec_TailPad>
-// CIR: cir.load align(8) %[[RESULT_P]] : !cir.ptr<!rec_TailPad>, !rec_TailPad
+// CIR: cir.copy %[[RESULT_P]] align(8) to %{{.+}} align(8) : !cir.ptr<!rec_TailPad>
// LLVM-LABEL: define dso_local noundef i64 @_Z16varargs_tail_padiz(i32 noundef %{{.*}}, ...)
// LLVMCIR: %[[GP:.+]] = load i32, ptr %{{.+}}, align 4
@@ -172,5 +170,4 @@ long varargs_tail_pad(int count, ...) {
// OGCG: store i64 %[[VAL]], ptr %[[DST]], align 8
// LLVMCIR: %[[ADDR:.+]] = phi ptr [ %{{.+}}, %{{.+}} ], [ %[[TEMP]], %{{.+}} ]
// OGCG: %[[ADDR:.+]] = phi ptr [ %[[TEMP]], %{{.+}} ], [ %{{.+}}, %{{.+}} ]
-// LLVMCIR: load %struct.TailPad, ptr %[[ADDR]], align 8
-// OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.+}}, ptr align 8 %[[ADDR]], i64 16, i1 false)
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{.+}}, ptr align 8 %[[ADDR]], i64 16, i1 false)
diff --git a/clang/test/CIR/CodeGenCoroutines/coro-agg.cpp b/clang/test/CIR/CodeGenCoroutines/coro-agg.cpp
index a966537abf9189..243a4d552df70d 100644
--- a/clang/test/CIR/CodeGenCoroutines/coro-agg.cpp
+++ b/clang/test/CIR/CodeGenCoroutines/coro-agg.cpp
@@ -36,8 +36,7 @@ coro_t aggregate_coawait_expr() {
// CIR: %[[RES:.*]] = cir.call @_ZN1B12await_resumeEv(%{{.*}})
// CIR: cir.store %[[RES]], %[[TMP:.*]] : !u64i, !cir.ptr<!u64i>
// CIR: %[[CAST:.*]] = cir.cast bitcast %[[TMP]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_B>
- // CIR: %[[LOAD:.*]] = cir.load %[[CAST]] : !cir.ptr<!rec_B>, !rec_B
- // CIR: cir.store align(4) %[[LOAD]], %[[VAL]] : !rec_B, !cir.ptr<!rec_B>
+ // CIR: cir.copy %[[CAST]] align(8) to %[[VAL]] align(4) : !cir.ptr<!rec_B>
// CIR: cir.yield
// CIR: },)
@@ -74,8 +73,7 @@ coro_t aggregate_coyield_expr() {
// CIR: %[[RES:.*]] = cir.call @_ZN1B12await_resumeEv(%{{.*}})
// CIR: cir.store %[[RES]], %[[TMP:.*]] : !u64i, !cir.ptr<!u64i>
// CIR: %[[CAST:.*]] = cir.cast bitcast %[[TMP]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_B>
- // CIR: %[[LOAD:.*]] = cir.load %[[CAST]] : !cir.ptr<!rec_B>, !rec_B
- // CIR: cir.store align(4) %[[LOAD]], %[[VAL]] : !rec_B, !cir.ptr<!rec_B>
+ // CIR: cir.copy %[[CAST]] align(8) to %[[VAL]] align(4) : !cir.ptr<!rec_B>
// CIR: cir.yield
// CIR: },)
diff --git a/clang/test/CIR/CodeGenOpenACC/private-clause-coerced-ctor.cpp b/clang/test/CIR/CodeGenOpenACC/private-clause-coerced-ctor.cpp
index 2c5a7f557f3859..eb12406738285b 100644
--- a/clang/test/CIR/CodeGenOpenACC/private-clause-coerced-ctor.cpp
+++ b/clang/test/CIR/CodeGenOpenACC/private-clause-coerced-ctor.cpp
@@ -1,15 +1,12 @@
// RUN: %clang_cc1 -fopenacc -triple x86_64-unknown-linux-gnu -fclangir \
// RUN: -fclangir-call-conv-lowering -emit-cir %s -o - | FileCheck %s
-struct Pair {
- long a, b;
+struct Triple {
+ int a, b, c;
};
-// Pair is two eightbytes, so the constructor's argument is coerced and the
-// call needs a coercion slot. The recipe body is not inside a cir.func when
-// CallConvLowering runs, so the slot goes at the start of the init region.
struct HasCoercedCtor {
- HasCoercedCtor(Pair p = Pair{});
+ HasCoercedCtor(Triple t = Triple{});
~HasCoercedCtor();
};
@@ -19,12 +16,16 @@ void privatized() {
;
}
+// A call outside a cir.func gets its coercion slot at the recipe's start.
// CHECK: acc.private.recipe @privatization__ZTS14HasCoercedCtor : !cir.ptr<!rec_HasCoercedCtor> init {
// CHECK-NEXT: ^bb0(%{{[^,)]+}}: !cir.ptr<!rec_HasCoercedCtor>
-// CHECK-NEXT: %[[COERCE:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!rec_Pair>
+// CHECK-NEXT: %[[COERCE:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!rec_anon_struct>
// CHECK-NEXT: cir.alloca "openacc.private.init"
-// CHECK: cir.store %{{.+}}, %[[COERCE]] : !rec_Pair, !cir.ptr<!rec_Pair>
-// CHECK: %[[VIEW:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!rec_Pair> -> !cir.ptr<!rec_anon_struct>
-// CHECK: %[[A:.*]] = cir.load %{{.+}} : !cir.ptr<!s64i>, !s64i
-// CHECK: %[[B:.*]] = cir.load %{{.+}} : !cir.ptr<!s64i>, !s64i
-// CHECK: cir.call @_ZN14HasCoercedCtorC1E4Pair(%{{.+}}, %[[A]], %[[B]]) : (!cir.ptr<!rec_HasCoercedCtor> {{.*}}, !s64i, !s64i) -> ()
+// CHECK-NEXT: %[[TMP:.*]] = cir.alloca "agg.tmp{{[0-9]*}}" align(4) : !cir.ptr<!rec_Triple>
+// CHECK: %[[VIEW:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_Triple>
+// CHECK-NEXT: cir.copy %[[TMP]] align(4) to %[[VIEW]] align(8) : !cir.ptr<!rec_Triple>
+// CHECK-NEXT: %[[A_PTR:.*]] = cir.get_member %[[COERCE]][0] {{.*}} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+// CHECK-NEXT: %[[A:.*]] = cir.load %[[A_PTR]] : !cir.ptr<!u64i>, !u64i
+// CHECK-NEXT: %[[B_PTR:.*]] = cir.get_member %[[COERCE]][1] {{.*}} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!s32i>
+// CHECK-NEXT: %[[B:.*]] = cir.load %[[B_PTR]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.call @_ZN14HasCoercedCtorC1E6Triple(%{{.+}}, %[[A]], %[[B]]) : (!cir.ptr<!rec_HasCoercedCtor> {{.*}}, !u64i, !s32i) -> ()
diff --git a/clang/test/CIR/Transforms/abi-lowering/coerce-alloca-addrspace.cir b/clang/test/CIR/Transforms/abi-lowering/coerce-alloca-addrspace.cir
index f4ad7d8bc9e0a1..2556ab59989755 100644
--- a/clang/test/CIR/Transforms/abi-lowering/coerce-alloca-addrspace.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/coerce-alloca-addrspace.cir
@@ -88,7 +88,6 @@ cir.func @takes_dim3(%arg0: !rec_Dim3)
}
// CHECK: cir.func{{.*}} @takes_dim3(%[[D0:.*]]: !s64i, %[[D1:.*]]: !s32i)
-// CHECK: cir.alloca "coerce" {{.*}} : !cir.ptr<{{.*}}, target_address_space(5)>
// CHECK: %[[FSLOT:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<{{.*}}, target_address_space(5)>
// CHECK: %[[P0:.*]] = cir.get_member %[[FSLOT]][0] {{.*}} -> !cir.ptr<!s64i, target_address_space(5)>
// CHECK: cir.store %[[D0]], %[[P0]] : !s64i, !cir.ptr<!s64i, target_address_space(5)>
diff --git a/clang/test/CIR/Transforms/abi-lowering/coerce-int-to-record.cir b/clang/test/CIR/Transforms/abi-lowering/coerce-int-to-record.cir
index fb837b0654f5cf..71f129399eec1c 100644
--- a/clang/test/CIR/Transforms/abi-lowering/coerce-int-to-record.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/coerce-int-to-record.cir
@@ -51,7 +51,8 @@ module attributes {
// CHECK: %[[RET:.*]] = cir.call @returns_pair() : () -> !s64i
// CHECK: cir.store %[[RET]], %{{.*}} : !s64i, !cir.ptr<!s64i>
// CHECK: %[[RCAST:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!s64i> -> !cir.ptr<!rec_Pair>
- // CHECK: %[[REC:.*]] = cir.load %[[RCAST]] : !cir.ptr<!rec_Pair>, !rec_Pair
+ // CHECK: %[[R:.*]] = cir.alloca "r" {{.*}} : !cir.ptr<!rec_Pair>
+ // CHECK: cir.copy %[[RCAST]] align(8) to %[[R]] align(4) : !cir.ptr<!rec_Pair>
// CHECK: %[[MEMBER:.*]] = cir.get_member %{{.*}}[0] {name = "first"} : !cir.ptr<!rec_Pair> -> !cir.ptr<!s32i>
}
diff --git a/clang/test/CIR/Transforms/abi-lowering/coerce-record-return-from-memory.cir b/clang/test/CIR/Transforms/abi-lowering/coerce-record-return-from-memory.cir
index 09e3099278c41b..a947627c3bfabb 100644
--- a/clang/test/CIR/Transforms/abi-lowering/coerce-record-return-from-memory.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/coerce-record-return-from-memory.cir
@@ -1,4 +1,5 @@
-// RUN: cir-opt %s -cir-call-conv-lowering="classification-attr=test_classify" \
+// RUN: cir-opt %s -split-input-file -verify-diagnostics \
+// RUN: -cir-call-conv-lowering="classification-attr=test_classify" \
// RUN: | FileCheck %s
!s8i = !cir.int<s, 8>
@@ -303,7 +304,8 @@ module attributes {
// CHECK-NEXT: %[[COERCED:.*]] = cir.load %[[SLOT]] : !cir.ptr<!s64i>, !s64i
// CHECK-NEXT: cir.return %[[COERCED]] : !s64i
- // A volatile load happens as written, so its value is stored into the slot.
+ // A volatile load stays volatile as a volatile copy into the slot, even
+ // from an alloca that could otherwise be read in place.
cir.func @returns_volatile() -> !rec_Pair64
attributes { test_classify = #ret_s64 } {
%0 = cir.alloca "__retval" align(8) : !cir.ptr<!rec_Pair64>
@@ -314,27 +316,9 @@ module attributes {
// CHECK-LABEL: cir.func{{.*}} @returns_volatile() -> !s64i
// CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_Pair64>
// CHECK-NEXT: %[[RETVAL:.*]] = cir.alloca "__retval" align(8) : !cir.ptr<!rec_Pair64>
- // CHECK-NEXT: %[[REC:.*]] = cir.load volatile %[[RETVAL]] : !cir.ptr<!rec_Pair64>, !rec_Pair64
- // CHECK-NEXT: cir.store %[[REC]], %[[SLOT]] : !rec_Pair64, !cir.ptr<!rec_Pair64>
+ // CHECK-NEXT: cir.copy %[[RETVAL]] align(8) to %[[SLOT]] align(8) volatile : !cir.ptr<!rec_Pair64>
// CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_Pair64> -> !cir.ptr<!s64i>
- // CHECK-NEXT: %[[COERCED:.*]] = cir.load %[[VIEW]] : !cir.ptr<!s64i>, !s64i
- // CHECK-NEXT: cir.return %[[COERCED]] : !s64i
-
- // An atomic load has to happen as written too.
- cir.func @returns_atomic() -> !rec_Pair64
- attributes { test_classify = #ret_s64 } {
- %0 = cir.alloca "__retval" align(8) : !cir.ptr<!rec_Pair64>
- %1 = cir.load atomic(seq_cst) %0 : !cir.ptr<!rec_Pair64>, !rec_Pair64
- cir.return %1 : !rec_Pair64
- }
-
- // CHECK-LABEL: cir.func{{.*}} @returns_atomic() -> !s64i
- // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_Pair64>
- // CHECK-NEXT: %[[RETVAL:.*]] = cir.alloca "__retval" align(8) : !cir.ptr<!rec_Pair64>
- // CHECK-NEXT: %[[REC:.*]] = cir.load atomic(seq_cst) %[[RETVAL]] : !cir.ptr<!rec_Pair64>, !rec_Pair64
- // CHECK-NEXT: cir.store %[[REC]], %[[SLOT]] : !rec_Pair64, !cir.ptr<!rec_Pair64>
- // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_Pair64> -> !cir.ptr<!s64i>
- // CHECK-NEXT: %[[COERCED:.*]] = cir.load %[[VIEW]] : !cir.ptr<!s64i>, !s64i
+ // CHECK-NEXT: %[[COERCED:.*]] = cir.load align(8) %[[VIEW]] : !cir.ptr<!s64i>, !s64i
// CHECK-NEXT: cir.return %[[COERCED]] : !s64i
// A record read through a non-alloca pointer has its bytes copied.
@@ -366,3 +350,31 @@ module attributes {
// CHECK-NEXT: cir.return %[[COERCED]] : !rec_TwoI64
}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_Pair64 = !cir.struct<"Pair64" {data !s64i, data !s64i}>
+
+#ret_s64 = {
+ return = { kind = "direct", coerced_type = !s64i },
+ args = [ ]
+}
+
+module attributes {
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+
+ // An atomic load has to happen as written, and a copy cannot keep it
+ // atomic.
+ cir.func @returns_atomic() -> !rec_Pair64
+ attributes { test_classify = #ret_s64 } {
+ %0 = cir.alloca "__retval" align(8) : !cir.ptr<!rec_Pair64>
+ %1 = cir.load atomic(seq_cst) %0 : !cir.ptr<!rec_Pair64>, !rec_Pair64
+ // expected-error @below {{atomic load of a record operand not yet implemented in CallConvLowering}}
+ cir.return %1 : !rec_Pair64
+ }
+
+}
diff --git a/clang/test/CIR/Transforms/abi-lowering/coerce-record-return-larger.cir b/clang/test/CIR/Transforms/abi-lowering/coerce-record-return-larger.cir
index 8b2798bf979c35..f080569a409fa4 100644
--- a/clang/test/CIR/Transforms/abi-lowering/coerce-record-return-larger.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/coerce-record-return-larger.cir
@@ -57,7 +57,8 @@ module attributes {
// CHECK: %[[RET:.*]] = cir.call @returns_vec3() : () -> !rec_TwoI64
// CHECK: cir.store %[[RET]], %{{.*}} : !rec_TwoI64, !cir.ptr<!rec_TwoI64>
// CHECK: %[[RCAST:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!rec_TwoI64> -> !cir.ptr<!rec_Vec3>
- // CHECK: %[[REC:.*]] = cir.load %[[RCAST]] : !cir.ptr<!rec_Vec3>, !rec_Vec3
+ // CHECK: %[[R:.*]] = cir.alloca "r" {{.*}} : !cir.ptr<!rec_Vec3>
+ // CHECK: cir.copy %[[RCAST]] align(8) to %[[R]] align(4) : !cir.ptr<!rec_Vec3>
// CHECK: %[[MEMBER:.*]] = cir.get_member %{{.*}}[0] {name = "first"} : !cir.ptr<!rec_Vec3> -> !cir.ptr<!s32i>
}
diff --git a/clang/test/CIR/Transforms/abi-lowering/coerce-record-to-int.cir b/clang/test/CIR/Transforms/abi-lowering/coerce-record-to-int.cir
index 3cca31ac3a4af9..bca69d3a89bad3 100644
--- a/clang/test/CIR/Transforms/abi-lowering/coerce-record-to-int.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/coerce-record-to-int.cir
@@ -33,9 +33,8 @@ module attributes {
// CHECK: %[[SLOT:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!s64i>
// CHECK: cir.store %[[ARG]], %[[SLOT]] : !s64i, !cir.ptr<!s64i>
// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!s64i> -> !cir.ptr<!rec_Pair>
- // CHECK: %[[REC:.*]] = cir.load %[[CAST]] : !cir.ptr<!rec_Pair>, !rec_Pair
// CHECK: %[[P:.*]] = cir.alloca "p" {{.*}} : !cir.ptr<!rec_Pair>
- // CHECK: cir.store %[[REC]], %[[P]] : !rec_Pair, !cir.ptr<!rec_Pair>
+ // CHECK: cir.copy %[[CAST]] align(8) to %[[P]] align(4) : !cir.ptr<!rec_Pair>
cir.func @caller()
attributes { test_classify = #caller_no_args } {
@@ -47,10 +46,9 @@ module attributes {
// Call site coerces the !rec_Pair operand to !s64i through memory.
// CHECK: cir.func{{.*}} @caller()
- // CHECK: %[[VAL:.*]] = cir.load %{{.*}} : !cir.ptr<!rec_Pair>, !rec_Pair
- // CHECK: cir.store %[[VAL]], %{{.*}} : !rec_Pair, !cir.ptr<!rec_Pair>
- // CHECK: %[[ARGCAST:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!rec_Pair> -> !cir.ptr<!s64i>
- // CHECK: %[[ARGINT:.*]] = cir.load %[[ARGCAST]] : !cir.ptr<!s64i>, !s64i
+ // CHECK: %[[P:.*]] = cir.alloca "p" {{.*}} : !cir.ptr<!rec_Pair>
+ // CHECK: %[[ARGCAST:.*]] = cir.cast bitcast %[[P]] : !cir.ptr<!rec_Pair> -> !cir.ptr<!s64i>
+ // CHECK: %[[ARGINT:.*]] = cir.load align(4) %[[ARGCAST]] : !cir.ptr<!s64i>, !s64i
// CHECK: cir.call @takes_pair(%[[ARGINT]]) : (!s64i) -> ()
}
diff --git a/clang/test/CIR/Transforms/abi-lowering/coerce-record-to-record-via-memory.cir b/clang/test/CIR/Transforms/abi-lowering/coerce-record-to-record-via-memory.cir
index 98ad8dca446eec..830e973d940300 100644
--- a/clang/test/CIR/Transforms/abi-lowering/coerce-record-to-record-via-memory.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/coerce-record-to-record-via-memory.cir
@@ -34,9 +34,8 @@ module attributes {
// CHECK: %[[SLOT:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!rec_TwoI64>
// CHECK: cir.store %[[ARG]], %[[SLOT]] : !rec_TwoI64, !cir.ptr<!rec_TwoI64>
// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_TwoI64> -> !cir.ptr<!rec_Vec4>
- // CHECK: %[[REC:.*]] = cir.load %[[CAST]] : !cir.ptr<!rec_Vec4>, !rec_Vec4
// CHECK: %[[V:.*]] = cir.alloca "v" {{.*}} : !cir.ptr<!rec_Vec4>
- // CHECK: cir.store %[[REC]], %[[V]] : !rec_Vec4, !cir.ptr<!rec_Vec4>
+ // CHECK: cir.copy %[[CAST]] align(8) to %[[V]] align(4) : !cir.ptr<!rec_Vec4>
cir.func @caller()
attributes { test_classify = #caller_no_args } {
@@ -48,10 +47,9 @@ module attributes {
// Call site coerces the !rec_Vec4 operand to !rec_TwoI64 through memory.
// CHECK: cir.func{{.*}} @caller()
- // CHECK: %[[VAL:.*]] = cir.load %{{.*}} : !cir.ptr<!rec_Vec4>, !rec_Vec4
- // CHECK: cir.store %[[VAL]], %{{.*}} : !rec_Vec4, !cir.ptr<!rec_Vec4>
- // CHECK: %[[ARGCAST:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!rec_Vec4> -> !cir.ptr<!rec_TwoI64>
- // CHECK: %[[ARGREC:.*]] = cir.load %[[ARGCAST]] : !cir.ptr<!rec_TwoI64>, !rec_TwoI64
+ // CHECK: %[[V:.*]] = cir.alloca "v" {{.*}} : !cir.ptr<!rec_Vec4>
+ // CHECK: %[[ARGCAST:.*]] = cir.cast bitcast %[[V]] : !cir.ptr<!rec_Vec4> -> !cir.ptr<!rec_TwoI64>
+ // CHECK: %[[ARGREC:.*]] = cir.load align(4) %[[ARGCAST]] : !cir.ptr<!rec_TwoI64>, !rec_TwoI64
// CHECK: cir.call @takes_vec(%[[ARGREC]]) : (!rec_TwoI64) -> ()
}
diff --git a/clang/test/CIR/Transforms/abi-lowering/direct-flatten.cir b/clang/test/CIR/Transforms/abi-lowering/direct-flatten.cir
index d9d1442cf9fdd1..87bc751ab8e755 100644
--- a/clang/test/CIR/Transforms/abi-lowering/direct-flatten.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/direct-flatten.cir
@@ -66,8 +66,8 @@ module attributes {
} {
// Callee: the single TwoLong arg is replaced by two i64 block args. At entry
-// the scalars are stored into a coerced-struct alloca, then coerced back to
-// TwoLong for body uses.
+// the scalars are stored into a coerced-struct alloca, which the body reads as
+// TwoLong.
cir.func @takes_two_long(%arg0: !rec_TwoLong)
attributes { test_classify = #flatten_two } {
@@ -77,20 +77,18 @@ cir.func @takes_two_long(%arg0: !rec_TwoLong)
}
// CHECK: cir.func{{.*}} @takes_two_long(%[[F0:.*]]: !s64i, %[[F1:.*]]: !s64i)
-// Two ["coerce"] allocas: the coercion round-trip alloca and the flatSlot.
-// CHECK: %{{.*}} = cir.alloca "coerce"
+// One "coerce" alloca holds both the coerced struct and TwoLong.
// CHECK: %[[FSLOT:.*]] = cir.alloca "coerce"
-// Store each field block arg into the flat slot.
+// CHECK-NOT: cir.alloca "coerce"
+// Store each field block arg into the slot.
// CHECK: %[[P0:.*]] = cir.get_member %[[FSLOT]][0]
// CHECK: cir.store %[[F0]], %[[P0]] : !s64i, !cir.ptr<!s64i>
// CHECK: %[[P1:.*]] = cir.get_member %[[FSLOT]][1]
// CHECK: cir.store %[[F1]], %[[P1]] : !s64i, !cir.ptr<!s64i>
-// Reload the coerced struct, coerce to TwoLong, use in body.
-// CHECK: %{{.*}} = cir.load %[[FSLOT]]
-// CHECK: cir.cast bitcast %{{.*}} : !cir.ptr<{{.*}}> -> !cir.ptr<!rec_TwoLong>
-// CHECK: %[[TVAL:.*]] = cir.load %{{.*}} : !cir.ptr<!rec_TwoLong>, !rec_TwoLong
+// The spill of the parameter is a copy out of the slot viewed as TwoLong.
+// CHECK: %[[TVIEW:.*]] = cir.cast bitcast %[[FSLOT]] : !cir.ptr<{{.*}}> -> !cir.ptr<!rec_TwoLong>
// CHECK: %[[P:.*]] = cir.alloca "p" {{.*}} : !cir.ptr<!rec_TwoLong>
-// CHECK: cir.store %[[TVAL]], %[[P]] : !rec_TwoLong, !cir.ptr<!rec_TwoLong>
+// CHECK: cir.copy %[[TVIEW]] align(8) to %[[P]] align(8) : !cir.ptr<!rec_TwoLong>
// Forward declaration: two i64 wire args, no body.
@@ -100,8 +98,8 @@ cir.func private @takes_two_long_decl(!rec_TwoLong)
// CHECK: cir.func{{.*}} @takes_two_long_decl(!s64i, !s64i)
// CHECK-NOT: !rec_TwoLong
-// Caller: the TwoLong operand is coerced into a coerced-struct slot via a
-// memory round-trip, then each field is read from that slot with
+// Caller: the TwoLong operand's alloca is as large as the coerced struct, so
+// each field is read in place through a coerced-struct view of it with
// get_member + load and passed as a separate call argument.
cir.func @caller_two_long() attributes { test_classify = #passthrough } {
@@ -112,16 +110,13 @@ cir.func @caller_two_long() attributes { test_classify = #passthrough } {
}
// CHECK: cir.func{{.*}} @caller_two_long()
-// CHECK: %[[COERCE:.*]] = cir.alloca "coerce"
-// CHECK: %[[VAL:.*]] = cir.load %{{.*}} : !cir.ptr<!rec_TwoLong>, !rec_TwoLong
-// Store the source value into the coerce slot, then read each field through a
-// coerced-struct view with get_member + load.
-// CHECK: cir.store %[[VAL]], %[[COERCE]] : !rec_TwoLong, !cir.ptr<!rec_TwoLong>
-// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!rec_TwoLong> -> !cir.ptr<{{.*}}>
+// CHECK: %[[S:.*]] = cir.alloca "s"
+// CHECK-NOT: cir.alloca "coerce"
+// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[S]] : !cir.ptr<!rec_TwoLong> -> !cir.ptr<{{.*}}>
// CHECK: %[[P0:.*]] = cir.get_member %[[CAST]][0] {{.*}} -> !cir.ptr<!s64i>
-// CHECK: %[[F0:.*]] = cir.load %[[P0]] : !cir.ptr<!s64i>, !s64i
+// CHECK: %[[F0:.*]] = cir.load align(8) %[[P0]] : !cir.ptr<!s64i>, !s64i
// CHECK: %[[P1:.*]] = cir.get_member %[[CAST]][1] {{.*}} -> !cir.ptr<!s64i>
-// CHECK: %[[F1:.*]] = cir.load %[[P1]] : !cir.ptr<!s64i>, !s64i
+// CHECK: %[[F1:.*]] = cir.load align(8) %[[P1]] : !cir.ptr<!s64i>, !s64i
// CHECK: cir.call @takes_two_long(%[[F0]], %[[F1]]) : (!s64i, !s64i) -> ()
// Three-field flatten (CUDA dim3 with 3 equal-width fields → 3 scalar args).
@@ -133,9 +128,8 @@ cir.func @takes_three_int(%arg0: !rec_ThreeInt)
}
// CHECK: cir.func{{.*}} @takes_three_int(%[[A:.*]]: !s32i, %[[B:.*]]: !s32i, %[[C:.*]]: !s32i)
-// Two ["coerce"] allocas appear; the fields are stored via the second one.
-// CHECK: cir.alloca "coerce"
// CHECK: cir.alloca "coerce"
+// CHECK-NOT: cir.alloca "coerce"
// CHECK: %[[PA:.*]] = cir.get_member %{{.*}}[0]
// CHECK: cir.store %[[A]], %[[PA]] : !s32i, !cir.ptr<!s32i>
// CHECK: %[[PB:.*]] = cir.get_member %{{.*}}[1]
@@ -151,16 +145,15 @@ cir.func @caller_three_int() attributes { test_classify = #passthrough } {
}
// CHECK: cir.func{{.*}} @caller_three_int()
-// CHECK: %[[COERCE:.*]] = cir.alloca "coerce"
-// CHECK: %[[VAL:.*]] = cir.load %{{.*}} : !cir.ptr<!rec_ThreeInt>, !rec_ThreeInt
-// CHECK: cir.store %[[VAL]], %[[COERCE]] : !rec_ThreeInt, !cir.ptr<!rec_ThreeInt>
-// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[COERCE]] : !cir.ptr<!rec_ThreeInt> -> !cir.ptr<{{.*}}>
+// CHECK: %[[S:.*]] = cir.alloca "s"
+// CHECK-NOT: cir.alloca "coerce"
+// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[S]] : !cir.ptr<!rec_ThreeInt> -> !cir.ptr<{{.*}}>
// CHECK: %[[P0:.*]] = cir.get_member %[[CAST]][0] {{.*}} -> !cir.ptr<!s32i>
-// CHECK: %[[F0:.*]] = cir.load %[[P0]] : !cir.ptr<!s32i>, !s32i
+// CHECK: %[[F0:.*]] = cir.load align(4) %[[P0]] : !cir.ptr<!s32i>, !s32i
// CHECK: %[[P1:.*]] = cir.get_member %[[CAST]][1] {{.*}} -> !cir.ptr<!s32i>
-// CHECK: %[[F1:.*]] = cir.load %[[P1]] : !cir.ptr<!s32i>, !s32i
+// CHECK: %[[F1:.*]] = cir.load align(4) %[[P1]] : !cir.ptr<!s32i>, !s32i
// CHECK: %[[P2:.*]] = cir.get_member %[[CAST]][2] {{.*}} -> !cir.ptr<!s32i>
-// CHECK: %[[F2:.*]] = cir.load %[[P2]] : !cir.ptr<!s32i>, !s32i
+// CHECK: %[[F2:.*]] = cir.load align(4) %[[P2]] : !cir.ptr<!s32i>, !s32i
// CHECK: cir.call @takes_three_int(%[[F0]], %[[F1]], %[[F2]]) : (!s32i, !s32i, !s32i) -> ()
// Mixed-packing case (x86-64 SysV dim3 pattern): struct { uint x, y, z; }
@@ -173,9 +166,8 @@ cir.func @takes_dim3(%arg0: !rec_Dim3)
}
// CHECK: cir.func{{.*}} @takes_dim3(%[[D0:.*]]: !s64i, %[[D1:.*]]: !s32i)
-// Two ["coerce"] allocas appear; the {i64, i32} fields are stored into the second.
-// CHECK: cir.alloca "coerce"
// CHECK: cir.alloca "coerce"
+// CHECK-NOT: cir.alloca "coerce"
// CHECK: %{{.*}} = cir.get_member %{{.*}}[0]
// CHECK: cir.store %[[D0]], %{{.*}} : !s64i
// CHECK: %{{.*}} = cir.get_member %{{.*}}[1]
@@ -189,13 +181,13 @@ cir.func @caller_dim3() attributes { test_classify = #passthrough } {
}
// CHECK: cir.func{{.*}} @caller_dim3()
-// Coerce Dim3 into the {i64, i32} slot, then read the two fields from the
+// Copy Dim3 into the {i64, i32} slot, then read the two fields from the
// slot with get_member + load. The coerced struct is larger than Dim3, so
-// the slot is the coerced type and the bitcast is on the store (source) side.
+// the slot is the coerced type and the bitcast is on the copy's side.
// CHECK: %[[COERCE:.*]] = cir.alloca "coerce"
-// CHECK: %[[VAL:.*]] = cir.load %{{.*}} : !cir.ptr<!rec_Dim3>, !rec_Dim3
+// CHECK: %[[D:.*]] = cir.alloca "d"
// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[COERCE]] : {{.*}} -> !cir.ptr<!rec_Dim3>
-// CHECK: cir.store %[[VAL]], %[[CAST]] : !rec_Dim3, !cir.ptr<!rec_Dim3>
+// CHECK: cir.copy %[[D]] align(4) to %[[CAST]] align(8) : !cir.ptr<!rec_Dim3>
// CHECK: %[[P0:.*]] = cir.get_member %[[COERCE]][0] {{.*}} -> !cir.ptr<!s64i>
// CHECK: %[[E0:.*]] = cir.load %[[P0]] : !cir.ptr<!s64i>, !s64i
// CHECK: %[[P1:.*]] = cir.get_member %[[COERCE]][1] {{.*}} -> !cir.ptr<!s32i>
@@ -230,17 +222,16 @@ cir.func @takes_self_flat(%arg0: !rec_coerced_two)
}
// CHECK: cir.func{{.*}} @takes_self_flat(%[[A:.*]]: !s64i, %[[B:.*]]: !s64i)
-// Only one "coerce" reassembly slot, and the reloaded coerced struct is stored
-// straight into the parameter alloca with no bitcast -- the arg type already
-// equals the coerced struct type, so there is no back-coercion.
+// Only one "coerce" reassembly slot, and the parameter alloca is copied from
+// it with no bitcast -- the arg type already equals the coerced struct type,
+// so there is no back-coercion.
// CHECK: %[[SLOT:.*]] = cir.alloca "coerce"
// CHECK: %[[P0:.*]] = cir.get_member %[[SLOT]][0]
// CHECK: cir.store %[[A]], %[[P0]] : !s64i, !cir.ptr<!s64i>
// CHECK: %[[P1:.*]] = cir.get_member %[[SLOT]][1]
// CHECK: cir.store %[[B]], %[[P1]] : !s64i, !cir.ptr<!s64i>
-// CHECK: %[[RV:.*]] = cir.load %[[SLOT]] : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
// CHECK: %[[P:.*]] = cir.alloca "p"
-// CHECK: cir.store %[[RV]], %[[P]] : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
+// CHECK: cir.copy %[[SLOT]] align(8) to %[[P]] align(8) : !cir.ptr<!rec_anon_struct>
cir.func @caller_self_flat() attributes { test_classify = #passthrough } {
%0 = cir.alloca "s" align(8) : !cir.ptr<!rec_coerced_two>
diff --git a/clang/test/CIR/Transforms/abi-lowering/indirect-byval.cir b/clang/test/CIR/Transforms/abi-lowering/indirect-byval.cir
index 8c52a9781b90ce..610d29bd6c3274 100644
--- a/clang/test/CIR/Transforms/abi-lowering/indirect-byval.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/indirect-byval.cir
@@ -447,7 +447,7 @@ module attributes {
// CHECK: cir.func{{.*}} @takes_big_uses_arg(%[[PTR:.*]]: !cir.ptr<!rec_Big>
// CHECK-SAME: llvm.byval = !rec_Big, llvm.noundef
- // CHECK: %[[LOADED:.*]] = cir.load %[[PTR]] : !cir.ptr<!rec_Big>, !rec_Big
+ // CHECK: %[[LOADED:.*]] = cir.load align(8) %[[PTR]] : !cir.ptr<!rec_Big>, !rec_Big
// CHECK-NEXT: cir.return %[[LOADED]] : !rec_Big
// Two byval arguments: an operand that names no storage gets its own
@@ -465,7 +465,7 @@ module attributes {
// CHECK-SAME: llvm.byval = !rec_Big, llvm.noundef
// The entry holds one load and the return, so the unused first parameter
// got nothing.
- // CHECK-NEXT: %[[LOADED:.*]] = cir.load %[[B]] : !cir.ptr<!rec_Big>, !rec_Big
+ // CHECK-NEXT: %[[LOADED:.*]] = cir.load align(8) %[[B]] : !cir.ptr<!rec_Big>, !rec_Big
// CHECK-NEXT: cir.return %[[LOADED]] : !rec_Big
cir.func @caller_two(%s: !rec_Big, %t: !rec_Big) -> !rec_Big
diff --git a/clang/test/CIR/Transforms/abi-lowering/indirect-non-byval-forward-param.cir b/clang/test/CIR/Transforms/abi-lowering/indirect-non-byval-forward-param.cir
index 685611b3ee51af..1c653851889838 100644
--- a/clang/test/CIR/Transforms/abi-lowering/indirect-non-byval-forward-param.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/indirect-non-byval-forward-param.cir
@@ -588,11 +588,9 @@ module attributes {
// CHECK: cir.func{{.*}} @unloaded_param_to_coerced_direct(%[[PTR:.*]]: !cir.ptr<!rec_Two>
// CHECK-SAME: {llvm.align = 8 : i64, llvm.dereferenceable = 16 : i64, llvm.nofreeobj, llvm.noundef})
- // CHECK: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_Two>
- // CHECK: %[[VAL:.*]] = cir.load align(8) %[[PTR]] : !cir.ptr<!rec_Two>, !rec_Two
- // CHECK: cir.store %[[VAL]], %[[SLOT]] : !rec_Two, !cir.ptr<!rec_Two>
- // CHECK: %[[AS:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_Two> -> !cir.ptr<!s64i>
- // CHECK: %[[COERCED:.*]] = cir.load %[[AS]] : !cir.ptr<!s64i>, !s64i
+ // CHECK-NOT: cir.alloca
+ // CHECK: %[[AS:.*]] = cir.cast bitcast %[[PTR]] : !cir.ptr<!rec_Two> -> !cir.ptr<!s64i>
+ // CHECK: %[[COERCED:.*]] = cir.load align(8) %[[AS]] : !cir.ptr<!s64i>, !s64i
// CHECK: cir.call @takes_two_coerced(%[[COERCED]]) : (!s64i) -> ()
}
diff --git a/clang/test/CIR/Transforms/abi-lowering/indirect-return-sret.cir b/clang/test/CIR/Transforms/abi-lowering/indirect-return-sret.cir
index e3a569f21c336e..3c1336b3657d43 100644
--- a/clang/test/CIR/Transforms/abi-lowering/indirect-return-sret.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/indirect-return-sret.cir
@@ -90,7 +90,7 @@ module attributes {
// CHECK: cir.return %{{.*}} : !s32i
// A call whose result has more than one use cannot reuse a destination, so a
- // fresh sret slot is allocated and the result is loaded back out of it.
+ // fresh sret slot is allocated and each store of the result copies out of it.
cir.func @caller_multi() -> !s32i
attributes { test_classify = #caller_cls } {
%a = cir.alloca "a" align(8) : !cir.ptr<!rec_Big>
@@ -109,9 +109,8 @@ module attributes {
// CHECK: cir.call @returns_big(%[[SLOT]]) : (!cir.ptr<!rec_Big>
// CHECK-SAME: llvm.sret = !rec_Big
// CHECK-SAME: -> ()
- // CHECK: %[[LD:.*]] = cir.load %[[SLOT]] : !cir.ptr<!rec_Big>, !rec_Big
- // CHECK: cir.store %[[LD]], %[[A]] : !rec_Big, !cir.ptr<!rec_Big>
- // CHECK: cir.store %[[LD]], %[[B]] : !rec_Big, !cir.ptr<!rec_Big>
+ // CHECK-NEXT: cir.copy %[[SLOT]] align(8) to %[[A]] align(8) : !cir.ptr<!rec_Big>
+ // CHECK-NEXT: cir.copy %[[SLOT]] align(8) to %[[B]] align(8) : !cir.ptr<!rec_Big>
// sret composes with an Extend argument: the sret slot is prepended and the
// s8 argument is still extended to s32 with llvm.signext on both the callee
diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-aggregate-nyi.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-aggregate-nyi.cir
index 9466d541cd68ad..1d91d49032846e 100644
--- a/clang/test/CIR/Transforms/abi-lowering/x86_64-aggregate-nyi.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-aggregate-nyi.cir
@@ -12,35 +12,6 @@
[#cir.bitfield_decl<!cir.int<s, 72, bitint>, 70>]>},
padding = {!cir.array<!u8i x 7>}>
!rec_SVec = !cir.struct<"SVec" {data !cir.vector<4 x !cir.float>}>
-!rec_SX87 = !cir.struct<"SX87" {data !cir.vector<1 x !cir.long_double<!cir.f80>>}>
-!rec_UX87Bytes =
- !cir.union<"UX87Bytes" {data !cir.vector<1 x !cir.long_double<!cir.f80>>,
- data !cir.array<!u8i x 16>}>
-!rec_UX87InRec =
- !cir.union<"UX87InRec" {data !rec_SX87, data !cir.array<!u8i x 16>}>
-!rec_UX87InArr =
- !cir.union<"UX87InArr" {data !cir.array<!cir.vector<1 x !cir.long_double<!cir.f80>> x 1>,
- data !cir.array<!u8i x 16>}>
-!rec_SUX87 = !cir.struct<"SUX87" {data !rec_UX87Bytes}>
-!rec_UX87Twice =
- !cir.union<"UX87Twice" {data !cir.vector<1 x !cir.long_double<!cir.f80>>,
- data !cir.vector<1 x !cir.long_double<!cir.f80>>}>
-!rec_UX87Scalar =
- !cir.union<"UX87Scalar" {data !cir.long_double<!cir.f80>,
- data !cir.array<!u8i x 11>}>
-!rec_SLongDouble = !cir.struct<"SLongDouble" {data !cir.long_double<!cir.f80>}>
-!rec_UX87Nested =
- !cir.union<"UX87Nested" {data !rec_SLongDouble, data !cir.array<!u8i x 4>}>
-!rec_UEmptyX87 =
- !cir.union<"UEmptyX87" {data !s32i,
- empty !cir.array<!cir.long_double<!cir.f80> x 0>},
- padding = {!cir.array<!u8i x 16>}>
-!rec_UX87Displaced =
- !cir.union<"UX87Displaced" {data !rec_UEmptyX87,
- data !cir.long_double<!cir.f80>}>
-!rec_UX87Complex =
- !cir.union<"UX87Complex" {data !cir.complex<!cir.long_double<!cir.f80>>,
- data !cir.array<!u8i x 32>}>
!rec_UVec384 =
!cir.union<"UVec384" {data !cir.vector<4 x !cir.float>, data !cir.array<!s8i x 33>},
padding = {!cir.array<!u8i x 32>}>
@@ -197,69 +168,6 @@ module attributes {
// CHECK: not yet implemented for type '!cir.vector<16 x !cir.int<s, 2, bitint>>
- // A union is moved as a value of its storage type, so a member holding data
- // past the 10 bytes an x87 double extended value stores would lose it. A
- // member reaching byte 11 is refused next to one, and so is a double
- // extended value held any other way, in a record, a _Complex, a vector or an
- // array, even next to one of the same type. A record holding such a union
- // is refused with it.
- cir.func @take_union_x87_scalar(%arg0: !rec_UX87Scalar) {
- cir.return
- }
-
- // CHECK: not yet implemented for type '!cir.union<"UX87Scalar"
-
- cir.func @take_union_x87_nested(%arg0: !rec_UX87Nested) {
- cir.return
- }
-
- // CHECK: not yet implemented for type '!cir.union<"UX87Nested"
-
- // A member that ties the double extended member on alignment and size,
- // listed first, becomes the storage type in its place, so the union is
- // refused even though that member's data ends within the 10 bytes.
- cir.func @take_union_x87_displaced(%arg0: !rec_UX87Displaced) {
- cir.return
- }
-
- // CHECK: not yet implemented for type '!cir.union<"UX87Displaced"
-
- cir.func @take_union_x87_complex(%arg0: !rec_UX87Complex) {
- cir.return
- }
-
- // CHECK: not yet implemented for type '!cir.union<"UX87Complex"
-
- cir.func @take_union_x87_twice(%arg0: !rec_UX87Twice) {
- cir.return
- }
-
- // CHECK: not yet implemented for type '!cir.union<"UX87Twice"
-
- cir.func @take_union_x87_bytes(%arg0: !rec_UX87Bytes) {
- cir.return
- }
-
- // CHECK: not yet implemented for type '!cir.union<"UX87Bytes"
-
- cir.func @take_union_x87_in_rec(%arg0: !rec_UX87InRec) {
- cir.return
- }
-
- // CHECK: not yet implemented for type '!cir.union<"UX87InRec"
-
- cir.func @take_union_x87_in_arr(%arg0: !rec_UX87InArr) {
- cir.return
- }
-
- // CHECK: not yet implemented for type '!cir.union<"UX87InArr"
-
- cir.func @take_struct_union_x87(%arg0: !rec_SUX87) {
- cir.return
- }
-
- // CHECK: not yet implemented for type '!cir.struct<"SUX87"
-
// A definition (unlike a declaration) is refused: no incomplete-by-value
// definition can arise from valid C++, so this stays a hard reject rather
// than the declaration-only skip in x86_64-incomplete-record-declaration.cir.
diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-bitint.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-bitint.cir
index acb9fd9585b79c..95cd6005962d4f 100644
--- a/clang/test/CIR/Transforms/abi-lowering/x86_64-bitint.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-bitint.cir
@@ -198,18 +198,12 @@ module attributes {
// CHECK-SAME: : !cir.ptr<!rec_anon_struct>
// CHECK-NEXT: %[[VALSLOT:[0-9]+]] = cir.alloca "coerce" align(8)
// CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[FLAT:[0-9]+]] = cir.alloca "coerce" align(8)
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[E0:[0-9]+]] = cir.get_member %[[FLAT]][0]
+ // CHECK-NEXT: %[[E0:[0-9]+]] = cir.get_member %[[VALSLOT]][0]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg0, %[[E0]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[E1:[0-9]+]] = cir.get_member %[[FLAT]][1]
+ // CHECK-NEXT: %[[E1:[0-9]+]] = cir.get_member %[[VALSLOT]][1]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg1, %[[E1]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[PAIR:[0-9]+]] = cir.load %[[FLAT]]
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
- // CHECK-NEXT: cir.store %[[PAIR]], %[[VALSLOT]]
- // CHECK-SAME: : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CHECK-NEXT: %[[VALVIEW:[0-9]+]] = cir.cast bitcast %[[VALSLOT]]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!cir.int<s, 65, bitint>>
// CHECK-NEXT: %[[VAL:[0-9]+]] = cir.load %[[VALVIEW]]
@@ -226,15 +220,12 @@ module attributes {
// LLVM-SAME: i64 %[[A0:[0-9]+]], i64 %[[A1:[0-9]+]])
// LLVM-NEXT: %[[RETSLOT:[0-9]+]] = alloca { i64, i64 }
// LLVM-NEXT: %[[VALSLOT:[0-9]+]] = alloca { i64, i64 }
- // LLVM-NEXT: %[[FLAT:[0-9]+]] = alloca { i64, i64 }
// LLVM-NEXT: %[[E0:[0-9]+]] = getelementptr inbounds nuw { i64, i64 }
- // LLVM-SAME: , ptr %[[FLAT]], i32 0, i32 0
+ // LLVM-SAME: , ptr %[[VALSLOT]], i32 0, i32 0
// LLVM-NEXT: store i64 %[[A0]], ptr %[[E0]]
// LLVM-NEXT: %[[E1:[0-9]+]] = getelementptr inbounds nuw { i64, i64 }
- // LLVM-SAME: , ptr %[[FLAT]], i32 0, i32 1
+ // LLVM-SAME: , ptr %[[VALSLOT]], i32 0, i32 1
// LLVM-NEXT: store i64 %[[A1]], ptr %[[E1]]
- // LLVM-NEXT: %[[PAIR:[0-9]+]] = load { i64, i64 }, ptr %[[FLAT]]
- // LLVM-NEXT: store { i64, i64 } %[[PAIR]], ptr %[[VALSLOT]]
// LLVM-NEXT: %[[WIDE:[0-9]+]] = load i128, ptr %[[VALSLOT]]
// LLVM-NEXT: %[[NARROW:[0-9]+]] = trunc i128 %[[WIDE]] to i65
// LLVM-NEXT: %[[EXT:[0-9]+]] = sext i65 %[[NARROW]] to i128
@@ -253,18 +244,12 @@ module attributes {
// CHECK-SAME: : !cir.ptr<!rec_anon_struct>
// CHECK-NEXT: %[[VALSLOT:[0-9]+]] = cir.alloca "coerce" align(8)
// CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[FLAT:[0-9]+]] = cir.alloca "coerce" align(8)
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[E0:[0-9]+]] = cir.get_member %[[FLAT]][0]
+ // CHECK-NEXT: %[[E0:[0-9]+]] = cir.get_member %[[VALSLOT]][0]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg0, %[[E0]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[E1:[0-9]+]] = cir.get_member %[[FLAT]][1]
+ // CHECK-NEXT: %[[E1:[0-9]+]] = cir.get_member %[[VALSLOT]][1]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg1, %[[E1]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[PAIR:[0-9]+]] = cir.load %[[FLAT]]
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
- // CHECK-NEXT: cir.store %[[PAIR]], %[[VALSLOT]]
- // CHECK-SAME: : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CHECK-NEXT: %[[VALVIEW:[0-9]+]] = cir.cast bitcast %[[VALSLOT]]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!cir.int<u, 65, bitint>>
// CHECK-NEXT: %[[VAL:[0-9]+]] = cir.load %[[VALVIEW]]
@@ -281,15 +266,12 @@ module attributes {
// LLVM-SAME: i64 %[[A0:[0-9]+]], i64 %[[A1:[0-9]+]])
// LLVM-NEXT: %[[RETSLOT:[0-9]+]] = alloca { i64, i64 }
// LLVM-NEXT: %[[VALSLOT:[0-9]+]] = alloca { i64, i64 }
- // LLVM-NEXT: %[[FLAT:[0-9]+]] = alloca { i64, i64 }
// LLVM-NEXT: %[[E0:[0-9]+]] = getelementptr inbounds nuw { i64, i64 }
- // LLVM-SAME: , ptr %[[FLAT]], i32 0, i32 0
+ // LLVM-SAME: , ptr %[[VALSLOT]], i32 0, i32 0
// LLVM-NEXT: store i64 %[[A0]], ptr %[[E0]]
// LLVM-NEXT: %[[E1:[0-9]+]] = getelementptr inbounds nuw { i64, i64 }
- // LLVM-SAME: , ptr %[[FLAT]], i32 0, i32 1
+ // LLVM-SAME: , ptr %[[VALSLOT]], i32 0, i32 1
// LLVM-NEXT: store i64 %[[A1]], ptr %[[E1]]
- // LLVM-NEXT: %[[PAIR:[0-9]+]] = load { i64, i64 }, ptr %[[FLAT]]
- // LLVM-NEXT: store { i64, i64 } %[[PAIR]], ptr %[[VALSLOT]]
// LLVM-NEXT: %[[WIDE:[0-9]+]] = load i128, ptr %[[VALSLOT]]
// LLVM-NEXT: %[[NARROW:[0-9]+]] = trunc i128 %[[WIDE]] to i65
// LLVM-NEXT: %[[EXT:[0-9]+]] = zext i65 %[[NARROW]] to i128
@@ -308,18 +290,12 @@ module attributes {
// CHECK-SAME: : !cir.ptr<!cir.int<s, 127, bitint>>
// CHECK-NEXT: %[[VALSLOT:[0-9]+]] = cir.alloca "coerce" align(8)
// CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[FLAT:[0-9]+]] = cir.alloca "coerce" align(8)
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[E0:[0-9]+]] = cir.get_member %[[FLAT]][0]
+ // CHECK-NEXT: %[[E0:[0-9]+]] = cir.get_member %[[VALSLOT]][0]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg0, %[[E0]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[E1:[0-9]+]] = cir.get_member %[[FLAT]][1]
+ // CHECK-NEXT: %[[E1:[0-9]+]] = cir.get_member %[[VALSLOT]][1]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg1, %[[E1]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[PAIR:[0-9]+]] = cir.load %[[FLAT]]
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
- // CHECK-NEXT: cir.store %[[PAIR]], %[[VALSLOT]]
- // CHECK-SAME: : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CHECK-NEXT: %[[VALVIEW:[0-9]+]] = cir.cast bitcast %[[VALSLOT]]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!cir.int<s, 127, bitint>>
// CHECK-NEXT: %[[VAL:[0-9]+]] = cir.load %[[VALVIEW]]
@@ -336,15 +312,12 @@ module attributes {
// LLVM-SAME: i64 %[[A0:[0-9]+]], i64 %[[A1:[0-9]+]])
// LLVM-NEXT: %[[RETSLOT:[0-9]+]] = alloca i128
// LLVM-NEXT: %[[VALSLOT:[0-9]+]] = alloca { i64, i64 }
- // LLVM-NEXT: %[[FLAT:[0-9]+]] = alloca { i64, i64 }
// LLVM-NEXT: %[[E0:[0-9]+]] = getelementptr inbounds nuw { i64, i64 }
- // LLVM-SAME: , ptr %[[FLAT]], i32 0, i32 0
+ // LLVM-SAME: , ptr %[[VALSLOT]], i32 0, i32 0
// LLVM-NEXT: store i64 %[[A0]], ptr %[[E0]]
// LLVM-NEXT: %[[E1:[0-9]+]] = getelementptr inbounds nuw { i64, i64 }
- // LLVM-SAME: , ptr %[[FLAT]], i32 0, i32 1
+ // LLVM-SAME: , ptr %[[VALSLOT]], i32 0, i32 1
// LLVM-NEXT: store i64 %[[A1]], ptr %[[E1]]
- // LLVM-NEXT: %[[PAIR:[0-9]+]] = load { i64, i64 }, ptr %[[FLAT]]
- // LLVM-NEXT: store { i64, i64 } %[[PAIR]], ptr %[[VALSLOT]]
// LLVM-NEXT: %[[WIDE:[0-9]+]] = load i128, ptr %[[VALSLOT]]
// LLVM-NEXT: %[[NARROW:[0-9]+]] = trunc i128 %[[WIDE]] to i127
// LLVM-NEXT: %[[EXT:[0-9]+]] = sext i127 %[[NARROW]] to i128
@@ -429,18 +402,12 @@ module attributes {
// CHECK-SAME: : !cir.ptr<!rec_anon_struct>
// CHECK-NEXT: %[[INSLOT:[0-9]+]] = cir.alloca "coerce" align(8)
// CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[FLAT:[0-9]+]] = cir.alloca "coerce" align(8)
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[I0:[0-9]+]] = cir.get_member %[[FLAT]][0]
+ // CHECK-NEXT: %[[I0:[0-9]+]] = cir.get_member %[[INSLOT]][0]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg0, %[[I0]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[I1:[0-9]+]] = cir.get_member %[[FLAT]][1]
+ // CHECK-NEXT: %[[I1:[0-9]+]] = cir.get_member %[[INSLOT]][1]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg1, %[[I1]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[INPAIR:[0-9]+]] = cir.load %[[FLAT]]
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
- // CHECK-NEXT: cir.store %[[INPAIR]], %[[INSLOT]]
- // CHECK-SAME: : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CHECK-NEXT: %[[INVIEW:[0-9]+]] = cir.cast bitcast %[[INSLOT]]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!cir.int<s, 65, bitint>>
// CHECK-NEXT: %[[VAL:[0-9]+]] = cir.load %[[INVIEW]]
@@ -513,18 +480,12 @@ module attributes {
// CHECK-SAME: : !cir.ptr<!cir.int<s, 33, bitint>>, !cir.int<s, 33, bitint>
// CHECK-NEXT: %[[P65SLOT:[0-9]+]] = cir.alloca "coerce" align(8)
// CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[FLAT:[0-9]+]] = cir.alloca "coerce" align(8)
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[E0:[0-9]+]] = cir.get_member %[[FLAT]][0]
+ // CHECK-NEXT: %[[E0:[0-9]+]] = cir.get_member %[[P65SLOT]][0]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg1, %[[E0]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[E1:[0-9]+]] = cir.get_member %[[FLAT]][1]
+ // CHECK-NEXT: %[[E1:[0-9]+]] = cir.get_member %[[P65SLOT]][1]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg2, %[[E1]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[PAIR:[0-9]+]] = cir.load %[[FLAT]]
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
- // CHECK-NEXT: cir.store %[[PAIR]], %[[P65SLOT]]
- // CHECK-SAME: : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CHECK-NEXT: %[[P65VIEW:[0-9]+]] = cir.cast bitcast %[[P65SLOT]]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!cir.int<s, 65, bitint>>
// CHECK-NEXT: %{{[0-9]+}} = cir.load %[[P65VIEW]]
@@ -541,15 +502,12 @@ module attributes {
// LLVM-NEXT: %{{[0-9]+}} = trunc i64 %[[W33]] to i33
// The second and third rebuild the 65-bit pair in order.
// LLVM-NEXT: %[[P65SLOT:[0-9]+]] = alloca { i64, i64 }, align 8
- // LLVM-NEXT: %[[FLAT:[0-9]+]] = alloca { i64, i64 }, align 8
// LLVM-NEXT: %[[E0:[0-9]+]] = getelementptr inbounds nuw { i64, i64 }
- // LLVM-SAME: , ptr %[[FLAT]], i32 0, i32 0
+ // LLVM-SAME: , ptr %[[P65SLOT]], i32 0, i32 0
// LLVM-NEXT: store i64 %[[A1]], ptr %[[E0]], align 8
// LLVM-NEXT: %[[E1:[0-9]+]] = getelementptr inbounds nuw { i64, i64 }
- // LLVM-SAME: , ptr %[[FLAT]], i32 0, i32 1
+ // LLVM-SAME: , ptr %[[P65SLOT]], i32 0, i32 1
// LLVM-NEXT: store i64 %[[A2]], ptr %[[E1]], align 8
- // LLVM-NEXT: %[[PAIR:[0-9]+]] = load { i64, i64 }, ptr %[[FLAT]], align 8
- // LLVM-NEXT: store { i64, i64 } %[[PAIR]], ptr %[[P65SLOT]], align 8
// LLVM-NEXT: %[[WIDE:[0-9]+]] = load i128, ptr %[[P65SLOT]], align 8
// LLVM-NEXT: %{{[0-9]+}} = trunc i128 %[[WIDE]] to i65
// LLVM-NEXT: ret void
@@ -573,18 +531,12 @@ module attributes {
// CHECK-SAME: : !cir.ptr<!cir.int<s, 33, bitint>>, !cir.int<s, 33, bitint>
// CHECK-NEXT: %[[V65SLOT:[0-9]+]] = cir.alloca "coerce" align(8)
// CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[FLAT:[0-9]+]] = cir.alloca "coerce" align(8)
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[E0:[0-9]+]] = cir.get_member %[[FLAT]][0]
+ // CHECK-NEXT: %[[E0:[0-9]+]] = cir.get_member %[[V65SLOT]][0]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg1, %[[E0]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[E1:[0-9]+]] = cir.get_member %[[FLAT]][1]
+ // CHECK-NEXT: %[[E1:[0-9]+]] = cir.get_member %[[V65SLOT]][1]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg2, %[[E1]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[PAIR:[0-9]+]] = cir.load %[[FLAT]]
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
- // CHECK-NEXT: cir.store %[[PAIR]], %[[V65SLOT]]
- // CHECK-SAME: : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
// CHECK-NEXT: %[[V65VIEW:[0-9]+]] = cir.cast bitcast %[[V65SLOT]]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!cir.int<s, 65, bitint>>
// CHECK-NEXT: %[[V65:[0-9]+]] = cir.load %[[V65VIEW]]
@@ -620,15 +572,12 @@ module attributes {
// LLVM-NEXT: %[[W33:[0-9]+]] = load i64, ptr %[[V33SLOT]], align 8
// LLVM-NEXT: %[[V33:[0-9]+]] = trunc i64 %[[W33]] to i33
// LLVM-NEXT: %[[V65SLOT:[0-9]+]] = alloca { i64, i64 }, align 8
- // LLVM-NEXT: %[[FLAT:[0-9]+]] = alloca { i64, i64 }, align 8
// LLVM-NEXT: %[[E0:[0-9]+]] = getelementptr inbounds nuw { i64, i64 }
- // LLVM-SAME: , ptr %[[FLAT]], i32 0, i32 0
+ // LLVM-SAME: , ptr %[[V65SLOT]], i32 0, i32 0
// LLVM-NEXT: store i64 %[[A1]], ptr %[[E0]], align 8
// LLVM-NEXT: %[[E1:[0-9]+]] = getelementptr inbounds nuw { i64, i64 }
- // LLVM-SAME: , ptr %[[FLAT]], i32 0, i32 1
+ // LLVM-SAME: , ptr %[[V65SLOT]], i32 0, i32 1
// LLVM-NEXT: store i64 %[[A2]], ptr %[[E1]], align 8
- // LLVM-NEXT: %[[PAIR:[0-9]+]] = load { i64, i64 }, ptr %[[FLAT]], align 8
- // LLVM-NEXT: store { i64, i64 } %[[PAIR]], ptr %[[V65SLOT]], align 8
// LLVM-NEXT: %[[WIDE:[0-9]+]] = load i128, ptr %[[V65SLOT]], align 8
// LLVM-NEXT: %[[V65:[0-9]+]] = trunc i128 %[[WIDE]] to i65
// Each callee register traces back to the value it carries.
@@ -677,9 +626,8 @@ module attributes {
// CHECK-NEXT: cir.store %arg0, %[[ARGSLOT]] : !s32i, !cir.ptr<!s32i>
// CHECK-NEXT: %[[ARGVIEW:[0-9]+]] = cir.cast bitcast %[[ARGSLOT]]
// CHECK-SAME: : !cir.ptr<!s32i> -> !cir.ptr<!rec_S>
- // CHECK-NEXT: %[[REC:[0-9]+]] = cir.load %[[ARGVIEW]] : !cir.ptr<!rec_S>, !rec_S
// CHECK-NEXT: %[[LOCAL:[0-9]+]] = cir.alloca "a" align(4) : !cir.ptr<!rec_S>
- // CHECK-NEXT: cir.store %[[REC]], %[[LOCAL]] : !rec_S, !cir.ptr<!rec_S>
+ // CHECK-NEXT: cir.copy %[[ARGVIEW]] align(4) to %[[LOCAL]] align(4) : !cir.ptr<!rec_S>
// CHECK-NEXT: %[[RETVIEW:[0-9]+]] = cir.cast bitcast %[[LOCAL]]
// CHECK-SAME: : !cir.ptr<!rec_S> -> !cir.ptr<!s32i>
// CHECK-NEXT: %[[RES:[0-9]+]] = cir.load align(4) %[[RETVIEW]] : !cir.ptr<!s32i>, !s32i
@@ -689,9 +637,8 @@ module attributes {
// LLVM-SAME: i32 %[[ARG:[0-9]+]])
// LLVM-NEXT: %[[ARGSLOT:[0-9]+]] = alloca i32, align 4
// LLVM-NEXT: store i32 %[[ARG]], ptr %[[ARGSLOT]], align 4
- // LLVM-NEXT: %[[REC:[0-9]+]] = load %struct.S, ptr %[[ARGSLOT]], align 4
// LLVM-NEXT: %[[LOCAL:[0-9]+]] = alloca %struct.S, align 4
- // LLVM-NEXT: store %struct.S %[[REC]], ptr %[[LOCAL]], align 4
+ // LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[LOCAL]], ptr align 4 %[[ARGSLOT]], i64 4, i1 false)
// LLVM-NEXT: %[[RES:[0-9]+]] = load i32, ptr %[[LOCAL]], align 4
// LLVM-NEXT: ret i32 %[[RES]]
@@ -703,36 +650,25 @@ module attributes {
// CHECK-LABEL: cir.func{{.*}} @take_p(%arg0: !u64i, %arg1: !u64i)
// CHECK-NEXT: %[[VALSLOT:[0-9]+]] = cir.alloca "coerce" align(8)
// CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[FLAT:[0-9]+]] = cir.alloca "coerce" align(8)
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[E0:[0-9]+]] = cir.get_member %[[FLAT]][0]
+ // CHECK-NEXT: %[[E0:[0-9]+]] = cir.get_member %[[VALSLOT]][0]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg0, %[[E0]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[E1:[0-9]+]] = cir.get_member %[[FLAT]][1]
+ // CHECK-NEXT: %[[E1:[0-9]+]] = cir.get_member %[[VALSLOT]][1]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
// CHECK-NEXT: cir.store %arg1, %[[E1]] : !u64i, !cir.ptr<!u64i>
- // CHECK-NEXT: %[[PAIR:[0-9]+]] = cir.load %[[FLAT]]
- // CHECK-SAME: : !cir.ptr<!rec_anon_struct>, !rec_anon_struct
- // CHECK-NEXT: cir.store %[[PAIR]], %[[VALSLOT]]
- // CHECK-SAME: : !rec_anon_struct, !cir.ptr<!rec_anon_struct>
- // CHECK-NEXT: %[[VIEW:[0-9]+]] = cir.cast bitcast %[[VALSLOT]]
+ // CHECK-NEXT: %{{[0-9]+}} = cir.cast bitcast %[[VALSLOT]]
// CHECK-SAME: : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_P>
- // CHECK-NEXT: %{{[0-9]+}} = cir.load %[[VIEW]] : !cir.ptr<!rec_P>, !rec_P
// CHECK-NEXT: cir.return
// LLVM-LABEL: define void @take_p(
// LLVM-SAME: i64 %[[A0:[0-9]+]], i64 %[[A1:[0-9]+]])
// LLVM-NEXT: %[[VALSLOT:[0-9]+]] = alloca { i64, i64 }, align 8
- // LLVM-NEXT: %[[FLAT:[0-9]+]] = alloca { i64, i64 }, align 8
// LLVM-NEXT: %[[E0:[0-9]+]] = getelementptr inbounds nuw { i64, i64 }
- // LLVM-SAME: , ptr %[[FLAT]], i32 0, i32 0
+ // LLVM-SAME: , ptr %[[VALSLOT]], i32 0, i32 0
// LLVM-NEXT: store i64 %[[A0]], ptr %[[E0]], align 8
// LLVM-NEXT: %[[E1:[0-9]+]] = getelementptr inbounds nuw { i64, i64 }
- // LLVM-SAME: , ptr %[[FLAT]], i32 0, i32 1
+ // LLVM-SAME: , ptr %[[VALSLOT]], i32 0, i32 1
// LLVM-NEXT: store i64 %[[A1]], ptr %[[E1]], align 8
- // LLVM-NEXT: %[[PAIR:[0-9]+]] = load { i64, i64 }, ptr %[[FLAT]], align 8
- // LLVM-NEXT: store { i64, i64 } %[[PAIR]], ptr %[[VALSLOT]], align 8
- // LLVM-NEXT: %{{[0-9]+}} = load %struct.P, ptr %[[VALSLOT]], align 16
// LLVM-NEXT: ret void
// A struct holding a _BitInt(128) coerces to a single i128.
@@ -743,16 +679,14 @@ module attributes {
// CHECK-LABEL: cir.func{{.*}} @take_w(%arg0: !s128i_bitint)
// CHECK-NEXT: %[[SLOT:[0-9]+]] = cir.alloca "coerce" align(8) : !cir.ptr<!s128i_bitint>
// CHECK-NEXT: cir.store %arg0, %[[SLOT]] : !s128i_bitint, !cir.ptr<!s128i_bitint>
- // CHECK-NEXT: %[[VIEW:[0-9]+]] = cir.cast bitcast %[[SLOT]]
+ // CHECK-NEXT: %{{[0-9]+}} = cir.cast bitcast %[[SLOT]]
// CHECK-SAME: : !cir.ptr<!s128i_bitint> -> !cir.ptr<!rec_W>
- // CHECK-NEXT: %{{[0-9]+}} = cir.load %[[VIEW]] : !cir.ptr<!rec_W>, !rec_W
// CHECK-NEXT: cir.return
// LLVM-LABEL: define void @take_w(
// LLVM-SAME: i128 %[[ARG:[0-9]+]])
// LLVM-NEXT: %[[SLOT:[0-9]+]] = alloca i128, align 8
// LLVM-NEXT: store i128 %[[ARG]], ptr %[[SLOT]], align 8
- // LLVM-NEXT: %{{[0-9]+}} = load %struct.W, ptr %[[SLOT]]
// LLVM-NEXT: ret void
// A trailing byte pushes the struct past two eightbytes, so it passes byval,
diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-empty-class.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-empty-class.cir
index 9f443a7e70e7e6..8b4ef992a1a086 100644
--- a/clang/test/CIR/Transforms/abi-lowering/x86_64-empty-class.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-empty-class.cir
@@ -148,8 +148,10 @@ module attributes {
// CHECK: %[[SLOT:.*]] = cir.alloca "coerce" align(4) : !cir.ptr<!rec_HasEmpty>
// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_HasEmpty> -> !cir.ptr<!s32i>
// CHECK: cir.store %arg0, %[[CAST]] : !s32i, !cir.ptr<!s32i>
- // CHECK: %{{.*}} = cir.load %[[SLOT]] : !cir.ptr<!rec_HasEmpty>, !rec_HasEmpty
+ // CHECK: %[[H:.*]] = cir.alloca "h" align(4) : !cir.ptr<!rec_HasEmpty>
+ // CHECK: cir.copy %[[SLOT]] align(4) to %[[H]] align(4) : !cir.ptr<!rec_HasEmpty>
// LLVM: define i32 @take_has_empty(i32 %{{.*}})
+ // LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %{{[^,]+}}, ptr align 4 %{{[^,]+}}, i64 8, i1 false)
// The empty member still takes layout space, so the int sits at offset 4 and
// the eightbyte spanning both coerces to i64.
diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-record-copy.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-record-copy.cir
new file mode 100644
index 00000000000000..6dc6fdd82d71ae
--- /dev/null
+++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-record-copy.cir
@@ -0,0 +1,1475 @@
+// RUN: cir-opt %s -split-input-file -verify-diagnostics -cir-call-conv-lowering=target=x86_64 | FileCheck %s
+// RUN: cir-opt %s -split-input-file -verify-diagnostics -cir-call-conv-lowering=target=x86_64 \
+// RUN: | FileCheck %s --check-prefix=MOVE
+// RUN: cir-opt %s -split-input-file -verify-diagnostics -cir-call-conv-lowering=target=x86_64 -cir-to-llvm -o - \
+// RUN: | cir-translate -mlir-to-llvmir --allow-unregistered-dialect \
+// RUN: | FileCheck %s --check-prefix=LLVM
+
+!s8i = !cir.int<s, 8>
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+!u8i = !cir.int<u, 8>
+!u32i = !cir.int<u, 32>
+!void = !cir.void
+!rec_UX87 = !cir.union<"UX87" {data !cir.long_double<!cir.f80>, data !cir.array<!u8i x 16>}>
+!rec_UX87Mem = !cir.union<"UX87Mem" {data !cir.complex<!cir.long_double<!cir.f80>>, data !cir.array<!u8i x 32>}>
+!rec_HS = !cir.struct<"HS" {data !s8i, data !s32i}>
+!rec_H = !cir.union<"H" {data !rec_HS, data !cir.array<!s32i x 2>}>
+!rec_HBS = !cir.struct<"HBS" {data !s8i, data !s64i, data !s64i, data !s64i}>
+!rec_HMem = !cir.union<"HMem" {data !rec_HBS, data !cir.array<!s32i x 8>}>
+!rec_S = !cir.struct<"S" {data !s32i, data !s32i}>
+!rec_SF = !cir.struct<"SF" {data !s64i, data !s32i}>
+!rec_S12 = !cir.struct<"S12" {data !s32i, data !s32i, data !s32i}>
+!rec_S16 = !cir.struct<"S16" {data !s32i, data !s32i, data !s32i, data !s32i}>
+!rec_U3 = !cir.union<"U3" {data !cir.array<!s8i x 3>}>
+!rec_SMem = !cir.struct<"SMem" {data !s64i, data !s64i, data !s64i}>
+!rec_TP = !cir.struct<"TP" {data !s32i, data !s8i, pad !cir.array<!u8i x 3>}>
+!rec_TPMem = !cir.struct<"TPMem" {data !s64i, data !s64i, data !s64i, data !s8i, pad !cir.array<!u8i x 7>}>
+!rec___va_list_tag = !cir.struct<"__va_list_tag" {data !u32i, data !u32i, data !cir.ptr<!void>, data !cir.ptr<!void>}>
+
+// CHECK-DAG: ![[PAIR:rec_anon_struct[0-9]*]] = !cir.struct<{data !u64i, data !u64i}>
+// CHECK-DAG: ![[SFPAIR:rec_anon_struct[0-9]*]] = !cir.struct<{data !s64i, data !s32i}>
+// CHECK-DAG: ![[S12PAIR:rec_anon_struct[0-9]*]] = !cir.struct<{data !u64i, data !s32i}>
+
+// Up to the constants at the end of the first module, no named record is
+// loaded or stored as a value.
+// MOVE-NOT: cir.load {{.*}}, !rec_{{[A-Z]}}
+// MOVE-NOT: cir.store {{.*}} : !rec_{{[A-Z]}}
+// MOVE-LABEL: cir.func @const_feed_x87(
+
+// LLVM-DAG: %union.UX87 = type { x86_fp80 }
+// LLVM-DAG: %union.H = type { %struct.HS }
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i8, dense<8>: vector<2xi64>>,
+ #dlti.dl_entry<i16, dense<16>: vector<2xi64>>,
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>,
+ #dlti.dl_entry<f32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<f64, dense<64>: vector<2xi64>>>
+} {
+
+ // Each callee is rewritten with its call sites in declaration order, so
+ // these call sites are rewritten before any definition below.
+ cir.func private @take_x87(!rec_UX87)
+ cir.func private @make_x87() -> !rec_UX87
+ cir.func private @take_h(!rec_H)
+ cir.func private @make_h() -> !rec_H
+ cir.func private @take_s(!rec_S)
+ cir.func private @make_s() -> !rec_S
+ cir.func private @take_sf(!rec_SF)
+ cir.func private @make_sf() -> !rec_SF
+ cir.func private @take_s12(!rec_S12)
+ cir.func private @take_s16(!rec_S16)
+ cir.func private @take_u3(!rec_U3)
+ cir.func private @take_x87mem(!rec_UX87Mem)
+ cir.func private @make_x87mem() -> !rec_UX87Mem
+ cir.func private @take_hmem(!rec_HMem)
+ cir.func private @make_hmem() -> !rec_HMem
+ cir.func private @take_smem(!rec_SMem)
+ cir.func private @make_smem() -> !rec_SMem
+ cir.func private @make_tp() -> !rec_TP
+ cir.func private @make_tpmem() -> !rec_TPMem
+
+ // CHECK: cir.func private @take_x87(!u64i, !u64i)
+ // CHECK: cir.func private @make_x87() -> ![[PAIR]]
+ // CHECK: cir.func private @take_h(!u64i)
+ // CHECK: cir.func private @make_h() -> !u64i
+ // CHECK: cir.func private @take_s(!u64i)
+ // CHECK: cir.func private @make_s() -> !u64i
+ // CHECK: cir.func private @take_sf(!s64i, !s32i)
+ // CHECK: cir.func private @make_sf() -> ![[SFPAIR]]
+ // CHECK: cir.func private @take_s12(!u64i, !s32i)
+ // CHECK: cir.func private @take_s16(!u64i, !u64i)
+ // CHECK: cir.func private @take_u3(!cir.int<u, 24>)
+ // CHECK: cir.func private @take_x87mem(!cir.ptr<!rec_UX87Mem> {llvm.align = 16 : i64, llvm.byval = !rec_UX87Mem, llvm.noundef})
+ // CHECK: cir.func private @make_x87mem(!cir.ptr<!rec_UX87Mem> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.sret = !rec_UX87Mem, llvm.writable})
+ // CHECK: cir.func private @take_hmem(!cir.ptr<!rec_HMem> {llvm.align = 8 : i64, llvm.byval = !rec_HMem, llvm.noundef})
+ // CHECK: cir.func private @make_hmem(!cir.ptr<!rec_HMem> {llvm.align = 8 : i64, llvm.dead_on_unwind, llvm.sret = !rec_HMem, llvm.writable})
+ // CHECK: cir.func private @take_smem(!cir.ptr<!rec_SMem> {llvm.align = 8 : i64, llvm.byval = !rec_SMem, llvm.noundef})
+ // CHECK: cir.func private @make_smem(!cir.ptr<!rec_SMem> {llvm.align = 8 : i64, llvm.dead_on_unwind, llvm.sret = !rec_SMem, llvm.writable})
+ // CHECK: cir.func private @make_tp() -> !u64i
+ // CHECK: cir.func private @make_tpmem(!cir.ptr<!rec_TPMem> {llvm.align = 8 : i64, llvm.dead_on_unwind, llvm.sret = !rec_TPMem, llvm.writable})
+
+ // A spilled parameter is copied out of its coercion slot.
+ cir.func @param_spill_x87(%arg0: !rec_UX87) {
+ %0 = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87>
+ cir.store %arg0, %0 : !rec_UX87, !cir.ptr<!rec_UX87>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @param_spill_x87(%arg0: !u64i, %arg1: !u64i)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[SLOT]][0] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.store %arg0, %[[LOP]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[SLOT]][1] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.store %arg1, %[[HIP]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[SPILL:.*]] = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(16) to %[[SPILL]] align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.return
+
+ cir.func @param_spill_h(%arg0: !rec_H) {
+ %0 = cir.alloca "h" align(4) : !cir.ptr<!rec_H>
+ cir.store %arg0, %0 : !rec_H, !cir.ptr<!rec_H>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @param_spill_h(%arg0: !u64i)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.store %arg0, %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[SPILL:.*]] = cir.alloca "h" align(4) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %[[SPILL]] align(4) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.return
+
+ cir.func @param_spill_s(%arg0: !rec_S) {
+ %0 = cir.alloca "s" align(4) : !cir.ptr<!rec_S>
+ cir.store %arg0, %0 : !rec_S, !cir.ptr<!rec_S>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @param_spill_s(%arg0: !u64i)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.store %arg0, %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_S>
+ // CHECK-NEXT: %[[SPILL:.*]] = cir.alloca "s" align(4) : !cir.ptr<!rec_S>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %[[SPILL]] align(4) : !cir.ptr<!rec_S>
+ // CHECK-NEXT: cir.return
+
+ cir.func @param_spill_sf(%arg0: !rec_SF) {
+ %0 = cir.alloca "s" align(8) : !cir.ptr<!rec_SF>
+ cir.store %arg0, %0 : !rec_SF, !cir.ptr<!rec_SF>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @param_spill_sf(%arg0: !s64i, %arg1: !s32i)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<![[SFPAIR]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[SLOT]][0] {name = ""} : !cir.ptr<![[SFPAIR]]> -> !cir.ptr<!s64i>
+ // CHECK-NEXT: cir.store %arg0, %[[LOP]] : !s64i, !cir.ptr<!s64i>
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[SLOT]][1] {name = ""} : !cir.ptr<![[SFPAIR]]> -> !cir.ptr<!s32i>
+ // CHECK-NEXT: cir.store %arg1, %[[HIP]] : !s32i, !cir.ptr<!s32i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<![[SFPAIR]]> -> !cir.ptr<!rec_SF>
+ // CHECK-NEXT: %[[SPILL:.*]] = cir.alloca "s" align(8) : !cir.ptr<!rec_SF>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %[[SPILL]] align(8) : !cir.ptr<!rec_SF>
+ // CHECK-NEXT: cir.return
+
+ // An unused parameter still fills its slot and nothing reads it.
+ cir.func @param_unused_x87(%arg0: !rec_UX87) {
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @param_unused_x87(%arg0: !u64i, %arg1: !u64i)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[SLOT]][0] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.store %arg0, %[[LOP]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[SLOT]][1] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.store %arg1, %[[HIP]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.cast bitcast %[[SLOT]] : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.return
+
+ cir.func @param_unused_h(%arg0: !rec_H) {
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @param_unused_h(%arg0: !u64i)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.store %arg0, %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.return
+
+ cir.func @param_unused_s(%arg0: !rec_S) {
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @param_unused_s(%arg0: !u64i)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.store %arg0, %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_S>
+ // CHECK-NEXT: cir.return
+
+ // The call is rewritten first and stores the parameter into an argument
+ // slot. The parameter's rewrite turns that store into a copy.
+ cir.func @param_forward_x87(%arg0: !rec_UX87) {
+ cir.call @take_x87(%arg0) : (!rec_UX87) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @param_forward_x87(%arg0: !u64i, %arg1: !u64i)
+ // CHECK: %[[VIEW:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[ARG:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(16) to %[[ARG]] align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[ARGVIEW:.*]] = cir.cast bitcast %[[ARG]] : !cir.ptr<!rec_UX87> -> !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[ARGVIEW]][0] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[LO:.*]] = cir.load %[[LOP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[ARGVIEW]][1] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[HI:.*]] = cir.load %[[HIP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_x87(%[[LO]], %[[HI]]) : (!u64i, !u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ cir.func @param_forward_h(%arg0: !rec_H) {
+ cir.call @take_h(%arg0) : (!rec_H) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @param_forward_h(%arg0: !u64i)
+ // CHECK: %[[VIEW:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[ARG:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %[[ARG]] align(4) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[ARGVIEW:.*]] = cir.cast bitcast %[[ARG]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[V:.*]] = cir.load %[[ARGVIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_h(%[[V]]) : (!u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ cir.func @param_forward_sf(%arg0: !rec_SF) {
+ cir.call @take_sf(%arg0) : (!rec_SF) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @param_forward_sf(%arg0: !s64i, %arg1: !s32i)
+ // CHECK: %[[VIEW:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<![[SFPAIR]]> -> !cir.ptr<!rec_SF>
+ // CHECK-NEXT: %[[ARG:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_SF>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %[[ARG]] align(8) : !cir.ptr<!rec_SF>
+ // CHECK-NEXT: %[[ARGVIEW:.*]] = cir.cast bitcast %[[ARG]] : !cir.ptr<!rec_SF> -> !cir.ptr<![[SFPAIR]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[ARGVIEW]][0] {name = ""} : !cir.ptr<![[SFPAIR]]> -> !cir.ptr<!s64i>
+ // CHECK-NEXT: %[[LO:.*]] = cir.load %[[LOP]] : !cir.ptr<!s64i>, !s64i
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[ARGVIEW]][1] {name = ""} : !cir.ptr<![[SFPAIR]]> -> !cir.ptr<!s32i>
+ // CHECK-NEXT: %[[HI:.*]] = cir.load %[[HIP]] : !cir.ptr<!s32i>, !s32i
+ // CHECK-NEXT: cir.call @take_sf(%[[LO]], %[[HI]]) : (!s64i, !s32i) -> ()
+ // CHECK-NEXT: cir.return
+
+ // A returned parameter is read in place out of its own coercion slot.
+ cir.func @param_return_x87(%arg0: !rec_UX87) -> !rec_UX87 {
+ cir.return %arg0 : !rec_UX87
+ }
+
+ // CHECK-LABEL: cir.func @param_return_x87(%arg0: !u64i, %arg1: !u64i)
+ // CHECK-SAME: -> ![[PAIR]]
+ // CHECK: %[[VIEW:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[RVIEW:.*]] = cir.cast bitcast %[[VIEW]] : !cir.ptr<!rec_UX87> -> !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[RET:.*]] = cir.load align(16) %[[RVIEW]] : !cir.ptr<![[PAIR]]>, ![[PAIR]]
+ // CHECK-NEXT: cir.return %[[RET]] : ![[PAIR]]
+
+ cir.func @param_return_h(%arg0: !rec_H) -> !rec_H {
+ cir.return %arg0 : !rec_H
+ }
+
+ // CHECK-LABEL: cir.func @param_return_h(%arg0: !u64i) -> !u64i
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.store %arg0, %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[RVIEW:.*]] = cir.cast bitcast %[[VIEW]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[RET:.*]] = cir.load align(8) %[[RVIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.return %[[RET]] : !u64i
+
+ cir.func @param_return_sf(%arg0: !rec_SF) -> !rec_SF {
+ cir.return %arg0 : !rec_SF
+ }
+
+ // CHECK-LABEL: cir.func @param_return_sf(%arg0: !s64i, %arg1: !s32i)
+ // CHECK-SAME: -> ![[SFPAIR]]
+ // CHECK: %[[VIEW:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<![[SFPAIR]]> -> !cir.ptr<!rec_SF>
+ // CHECK-NEXT: %[[RVIEW:.*]] = cir.cast bitcast %[[VIEW]] : !cir.ptr<!rec_SF> -> !cir.ptr<![[SFPAIR]]>
+ // CHECK-NEXT: %[[RET:.*]] = cir.load align(8) %[[RVIEW]] : !cir.ptr<![[SFPAIR]]>, ![[SFPAIR]]
+ // CHECK-NEXT: cir.return %[[RET]] : ![[SFPAIR]]
+
+ cir.func @ret_local_x87() -> !rec_UX87 {
+ %0 = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87>
+ %1 = cir.load %0 : !cir.ptr<!rec_UX87>, !rec_UX87
+ cir.return %1 : !rec_UX87
+ }
+
+ // CHECK-LABEL: cir.func @ret_local_x87()
+ // CHECK-SAME: -> ![[PAIR]]
+ // CHECK-NEXT: %[[U:.*]] = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[RVIEW:.*]] = cir.cast bitcast %[[U]] : !cir.ptr<!rec_UX87> -> !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[RET:.*]] = cir.load align(16) %[[RVIEW]] : !cir.ptr<![[PAIR]]>, ![[PAIR]]
+ // CHECK-NEXT: cir.return %[[RET]] : ![[PAIR]]
+
+ cir.func @ret_local_h() -> !rec_H {
+ %0 = cir.alloca "h" align(4) : !cir.ptr<!rec_H>
+ %1 = cir.load %0 : !cir.ptr<!rec_H>, !rec_H
+ cir.return %1 : !rec_H
+ }
+
+ // CHECK-LABEL: cir.func @ret_local_h() -> !u64i
+ // CHECK-NEXT: %[[H:.*]] = cir.alloca "h" align(4) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[RVIEW:.*]] = cir.cast bitcast %[[H]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[RET:.*]] = cir.load align(4) %[[RVIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.return %[[RET]] : !u64i
+
+ // An argument loaded from an alloca is read in place.
+ cir.func @call_inplace_x87() {
+ %0 = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87>
+ %1 = cir.load %0 : !cir.ptr<!rec_UX87>, !rec_UX87
+ cir.call @take_x87(%1) : (!rec_UX87) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @call_inplace_x87()
+ // CHECK-NEXT: %[[U:.*]] = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[U]] : !cir.ptr<!rec_UX87> -> !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[VIEW]][0] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[LO:.*]] = cir.load align(16) %[[LOP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[VIEW]][1] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[HI:.*]] = cir.load align(8) %[[HIP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_x87(%[[LO]], %[[HI]]) : (!u64i, !u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ cir.func @call_inplace_h() {
+ %0 = cir.alloca "h" align(4) : !cir.ptr<!rec_H>
+ %1 = cir.load %0 : !cir.ptr<!rec_H>, !rec_H
+ cir.call @take_h(%1) : (!rec_H) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @call_inplace_h()
+ // CHECK-NEXT: %[[H:.*]] = cir.alloca "h" align(4) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[H]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[V:.*]] = cir.load align(4) %[[VIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_h(%[[V]]) : (!u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ cir.func @call_inplace_s() {
+ %0 = cir.alloca "s" align(4) : !cir.ptr<!rec_S>
+ %1 = cir.load %0 : !cir.ptr<!rec_S>, !rec_S
+ cir.call @take_s(%1) : (!rec_S) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @call_inplace_s()
+ // CHECK-NEXT: %[[S:.*]] = cir.alloca "s" align(4) : !cir.ptr<!rec_S>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[S]] : !cir.ptr<!rec_S> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[V:.*]] = cir.load align(4) %[[VIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_s(%[[V]]) : (!u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ cir.func @call_inplace_sf() {
+ %0 = cir.alloca "s" align(8) : !cir.ptr<!rec_SF>
+ %1 = cir.load %0 : !cir.ptr<!rec_SF>, !rec_SF
+ cir.call @take_sf(%1) : (!rec_SF) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @call_inplace_sf()
+ // CHECK-NEXT: %[[S:.*]] = cir.alloca "s" align(8) : !cir.ptr<!rec_SF>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[S]] : !cir.ptr<!rec_SF> -> !cir.ptr<![[SFPAIR]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[VIEW]][0] {name = ""} : !cir.ptr<![[SFPAIR]]> -> !cir.ptr<!s64i>
+ // CHECK-NEXT: %[[LO:.*]] = cir.load align(8) %[[LOP]] : !cir.ptr<!s64i>, !s64i
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[VIEW]][1] {name = ""} : !cir.ptr<![[SFPAIR]]> -> !cir.ptr<!s32i>
+ // CHECK-NEXT: %[[HI:.*]] = cir.load align(8) %[[HIP]] : !cir.ptr<!s32i>, !s32i
+ // CHECK-NEXT: cir.call @take_sf(%[[LO]], %[[HI]]) : (!s64i, !s32i) -> ()
+ // CHECK-NEXT: cir.return
+
+ // Each field is read at the alignment the alloca guarantees at its offset.
+ cir.func @call_inplace_s16() {
+ %0 = cir.alloca "s" align(4) : !cir.ptr<!rec_S16>
+ %1 = cir.load %0 : !cir.ptr<!rec_S16>, !rec_S16
+ cir.call @take_s16(%1) : (!rec_S16) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @call_inplace_s16()
+ // CHECK-NEXT: %[[S:.*]] = cir.alloca "s" align(4) : !cir.ptr<!rec_S16>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[S]] : !cir.ptr<!rec_S16> -> !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[VIEW]][0] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[LO:.*]] = cir.load align(4) %[[LOP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[VIEW]][1] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[HI:.*]] = cir.load align(4) %[[HIP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_s16(%[[LO]], %[[HI]]) : (!u64i, !u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ // Storage that is not an alloca is copied into a slot first.
+ cir.func @call_copied_x87(%arg0: !cir.ptr<!rec_UX87>) {
+ %0 = cir.load %arg0 : !cir.ptr<!rec_UX87>, !rec_UX87
+ cir.call @take_x87(%0) : (!rec_UX87) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @call_copied_x87(%arg0: !cir.ptr<!rec_UX87>)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.copy %arg0 align(16) to %[[SLOT]] align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_UX87> -> !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[VIEW]][0] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[LO:.*]] = cir.load %[[LOP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[VIEW]][1] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[HI:.*]] = cir.load %[[HIP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_x87(%[[LO]], %[[HI]]) : (!u64i, !u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ cir.func @call_copied_h(%arg0: !cir.ptr<!rec_H>) {
+ %0 = cir.load %arg0 : !cir.ptr<!rec_H>, !rec_H
+ cir.call @take_h(%0) : (!rec_H) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @call_copied_h(%arg0: !cir.ptr<!rec_H>)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.copy %arg0 align(4) to %[[SLOT]] align(8) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[V:.*]] = cir.load align(8) %[[VIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_h(%[[V]]) : (!u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ // The coerced type is larger than the record, so the alloca is copied
+ // into a slot that holds both.
+ cir.func @call_small_u3() {
+ %0 = cir.alloca "u" align(1) : !cir.ptr<!rec_U3>
+ %1 = cir.load %0 : !cir.ptr<!rec_U3>, !rec_U3
+ cir.call @take_u3(%1) : (!rec_U3) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @call_small_u3()
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(4) : !cir.ptr<!rec_U3>
+ // CHECK-NEXT: %[[U:.*]] = cir.alloca "u" align(1) : !cir.ptr<!rec_U3>
+ // CHECK-NEXT: cir.copy %[[U]] align(1) to %[[SLOT]] align(4) : !cir.ptr<!rec_U3>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_U3> -> !cir.ptr<!cir.int<u, 24>>
+ // CHECK-NEXT: %[[V:.*]] = cir.load align(4) %[[VIEW]] : !cir.ptr<!cir.int<u, 24>>, !cir.int<u, 24>
+ // CHECK-NEXT: cir.call @take_u3(%[[V]]) : (!cir.int<u, 24>) -> ()
+ // CHECK-NEXT: cir.return
+
+ cir.func @call_small_s12() {
+ %0 = cir.alloca "s" align(4) : !cir.ptr<!rec_S12>
+ %1 = cir.load %0 : !cir.ptr<!rec_S12>, !rec_S12
+ cir.call @take_s12(%1) : (!rec_S12) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @call_small_s12()
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<![[S12PAIR]]>
+ // CHECK-NEXT: %[[S:.*]] = cir.alloca "s" align(4) : !cir.ptr<!rec_S12>
+ // CHECK-NEXT: %[[RVIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<![[S12PAIR]]> -> !cir.ptr<!rec_S12>
+ // CHECK-NEXT: cir.copy %[[S]] align(4) to %[[RVIEW]] align(8) : !cir.ptr<!rec_S12>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[SLOT]][0] {name = ""} : !cir.ptr<![[S12PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[LO:.*]] = cir.load %[[LOP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[SLOT]][1] {name = ""} : !cir.ptr<![[S12PAIR]]> -> !cir.ptr<!s32i>
+ // CHECK-NEXT: %[[HI:.*]] = cir.load %[[HIP]] : !cir.ptr<!s32i>, !s32i
+ // CHECK-NEXT: cir.call @take_s12(%[[LO]], %[[HI]]) : (!u64i, !s32i) -> ()
+ // CHECK-NEXT: cir.return
+
+ // A stored call result is copied out of its return slot.
+ cir.func @result_store_x87() {
+ %0 = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87>
+ %1 = cir.call @make_x87() : () -> !rec_UX87
+ cir.store %1, %0 : !rec_UX87, !cir.ptr<!rec_UX87>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @result_store_x87()
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[U:.*]] = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_x87() : () -> ![[PAIR]]
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : ![[PAIR]], !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(16) to %[[U]] align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.return
+
+ cir.func @result_store_h() {
+ %0 = cir.alloca "h" align(4) : !cir.ptr<!rec_H>
+ %1 = cir.call @make_h() : () -> !rec_H
+ cir.store %1, %0 : !rec_H, !cir.ptr<!rec_H>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @result_store_h()
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[H:.*]] = cir.alloca "h" align(4) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_h() : () -> !u64i
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %[[H]] align(4) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.return
+
+ cir.func @result_store_s() {
+ %0 = cir.alloca "s" align(4) : !cir.ptr<!rec_S>
+ %1 = cir.call @make_s() : () -> !rec_S
+ cir.store %1, %0 : !rec_S, !cir.ptr<!rec_S>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @result_store_s()
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[S:.*]] = cir.alloca "s" align(4) : !cir.ptr<!rec_S>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_s() : () -> !u64i
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_S>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %[[S]] align(4) : !cir.ptr<!rec_S>
+ // CHECK-NEXT: cir.return
+
+ cir.func @result_store_sf() {
+ %0 = cir.alloca "s" align(8) : !cir.ptr<!rec_SF>
+ %1 = cir.call @make_sf() : () -> !rec_SF
+ cir.store %1, %0 : !rec_SF, !cir.ptr<!rec_SF>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @result_store_sf()
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<![[SFPAIR]]>
+ // CHECK-NEXT: %[[S:.*]] = cir.alloca "s" align(8) : !cir.ptr<!rec_SF>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_sf() : () -> ![[SFPAIR]]
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : ![[SFPAIR]], !cir.ptr<![[SFPAIR]]>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<![[SFPAIR]]> -> !cir.ptr<!rec_SF>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %[[S]] align(8) : !cir.ptr<!rec_SF>
+ // CHECK-NEXT: cir.return
+
+ // A store into an array element is a copy into that element.
+ cir.func @result_stride_x87(%arg0: !cir.ptr<!rec_UX87>) {
+ %0 = cir.const #cir.int<3> : !s64i
+ %1 = cir.ptr_stride %arg0, %0 : (!cir.ptr<!rec_UX87>, !s64i) -> !cir.ptr<!rec_UX87>
+ %2 = cir.call @make_x87() : () -> !rec_UX87
+ cir.store align(16) %2, %1 : !rec_UX87, !cir.ptr<!rec_UX87>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @result_stride_x87(%arg0: !cir.ptr<!rec_UX87>)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[IDX:.*]] = cir.const #cir.int<3> : !s64i
+ // CHECK-NEXT: %[[ELT:.*]] = cir.ptr_stride %arg0, %[[IDX]] : (!cir.ptr<!rec_UX87>, !s64i) -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_x87() : () -> ![[PAIR]]
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : ![[PAIR]], !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(16) to %[[ELT]] align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.return
+
+ cir.func @result_stride_h(%arg0: !cir.ptr<!rec_H>) {
+ %0 = cir.const #cir.int<3> : !s64i
+ %1 = cir.ptr_stride %arg0, %0 : (!cir.ptr<!rec_H>, !s64i) -> !cir.ptr<!rec_H>
+ %2 = cir.call @make_h() : () -> !rec_H
+ cir.store align(4) %2, %1 : !rec_H, !cir.ptr<!rec_H>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @result_stride_h(%arg0: !cir.ptr<!rec_H>)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[IDX:.*]] = cir.const #cir.int<3> : !s64i
+ // CHECK-NEXT: %[[ELT:.*]] = cir.ptr_stride %arg0, %[[IDX]] : (!cir.ptr<!rec_H>, !s64i) -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_h() : () -> !u64i
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %[[ELT]] align(4) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.return
+
+ // The consumer is rewritten first and stores the result into an argument
+ // slot. The producer's rewrite turns that store into a copy.
+ cir.func @result_feed_x87() {
+ %0 = cir.call @make_x87() : () -> !rec_UX87
+ cir.call @take_x87(%0) : (!rec_UX87) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @result_feed_x87()
+ // CHECK-NEXT: %[[RSLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[ASLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_x87() : () -> ![[PAIR]]
+ // CHECK-NEXT: cir.store %[[RES]], %[[RSLOT]] : ![[PAIR]], !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[RVIEW:.*]] = cir.cast bitcast %[[RSLOT]] : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.copy %[[RVIEW]] align(16) to %[[ASLOT]] align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[AVIEW:.*]] = cir.cast bitcast %[[ASLOT]] : !cir.ptr<!rec_UX87> -> !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[AVIEW]][0] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[LO:.*]] = cir.load %[[LOP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[AVIEW]][1] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[HI:.*]] = cir.load %[[HIP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_x87(%[[LO]], %[[HI]]) : (!u64i, !u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ cir.func @result_feed_h() {
+ %0 = cir.call @make_h() : () -> !rec_H
+ cir.call @take_h(%0) : (!rec_H) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @result_feed_h()
+ // CHECK-NEXT: %[[RSLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[ASLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_h() : () -> !u64i
+ // CHECK-NEXT: cir.store %[[RES]], %[[RSLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[RVIEW:.*]] = cir.cast bitcast %[[RSLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.copy %[[RVIEW]] align(8) to %[[ASLOT]] align(4) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[AVIEW:.*]] = cir.cast bitcast %[[ASLOT]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[V:.*]] = cir.load %[[AVIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_h(%[[V]]) : (!u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ // The producer is rewritten first, so the return reads its slot in place.
+ cir.func @result_return_x87() -> !rec_UX87 {
+ %0 = cir.call @make_x87() : () -> !rec_UX87
+ cir.return %0 : !rec_UX87
+ }
+
+ // CHECK-LABEL: cir.func @result_return_x87()
+ // CHECK-SAME: -> ![[PAIR]]
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_x87() : () -> ![[PAIR]]
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : ![[PAIR]], !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[RVIEW:.*]] = cir.cast bitcast %[[VIEW]] : !cir.ptr<!rec_UX87> -> !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[RET:.*]] = cir.load align(16) %[[RVIEW]] : !cir.ptr<![[PAIR]]>, ![[PAIR]]
+ // CHECK-NEXT: cir.return %[[RET]] : ![[PAIR]]
+
+ cir.func @result_return_h() -> !rec_H {
+ %0 = cir.call @make_h() : () -> !rec_H
+ cir.return %0 : !rec_H
+ }
+
+ // CHECK-LABEL: cir.func @result_return_h() -> !u64i
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_h() : () -> !u64i
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[RVIEW:.*]] = cir.cast bitcast %[[VIEW]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[RET:.*]] = cir.load align(8) %[[RVIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.return %[[RET]] : !u64i
+
+ // A volatile store becomes a volatile copy.
+ cir.func @result_volatile_x87(%arg0: !cir.ptr<!rec_UX87>) {
+ %0 = cir.call @make_x87() : () -> !rec_UX87
+ cir.store volatile align(16) %0, %arg0 : !rec_UX87, !cir.ptr<!rec_UX87>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @result_volatile_x87(%arg0: !cir.ptr<!rec_UX87>)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_x87() : () -> ![[PAIR]]
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : ![[PAIR]], !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(16) to %arg0 align(16) volatile : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.return
+
+ cir.func @result_volatile_h(%arg0: !cir.ptr<!rec_H>) {
+ %0 = cir.call @make_h() : () -> !rec_H
+ cir.store volatile %0, %arg0 : !rec_H, !cir.ptr<!rec_H>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @result_volatile_h(%arg0: !cir.ptr<!rec_H>)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_h() : () -> !u64i
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %arg0 align(4) volatile : !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.return
+
+ // A result whose destination is loaded after the call is copied out of a
+ // return slot of its own.
+ cir.func @sret_nodom_x87mem(%arg0: !cir.ptr<!cir.ptr<!rec_UX87Mem>>) {
+ %0 = cir.call @make_x87mem() : () -> !rec_UX87Mem
+ %1 = cir.load %arg0 : !cir.ptr<!cir.ptr<!rec_UX87Mem>>, !cir.ptr<!rec_UX87Mem>
+ cir.store %0, %1 : !rec_UX87Mem, !cir.ptr<!rec_UX87Mem>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @sret_nodom_x87mem(%arg0: !cir.ptr<!cir.ptr<!rec_UX87Mem>>)
+ // CHECK-NEXT: %[[SRET:.*]] = cir.alloca "sret" align(16) : !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: cir.call @make_x87mem(%[[SRET]]) : (!cir.ptr<!rec_UX87Mem> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.sret = !rec_UX87Mem, llvm.writable}) -> ()
+ // CHECK-NEXT: %[[DEST:.*]] = cir.load %arg0 : !cir.ptr<!cir.ptr<!rec_UX87Mem>>, !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: cir.copy %[[SRET]] align(16) to %[[DEST]] align(16) : !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: cir.return
+
+ cir.func @sret_nodom_hmem(%arg0: !cir.ptr<!cir.ptr<!rec_HMem>>) {
+ %0 = cir.call @make_hmem() : () -> !rec_HMem
+ %1 = cir.load %arg0 : !cir.ptr<!cir.ptr<!rec_HMem>>, !cir.ptr<!rec_HMem>
+ cir.store %0, %1 : !rec_HMem, !cir.ptr<!rec_HMem>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @sret_nodom_hmem(%arg0: !cir.ptr<!cir.ptr<!rec_HMem>>)
+ // CHECK-NEXT: %[[SRET:.*]] = cir.alloca "sret" align(8) : !cir.ptr<!rec_HMem>
+ // CHECK-NEXT: cir.call @make_hmem(%[[SRET]]) : (!cir.ptr<!rec_HMem> {llvm.align = 8 : i64, llvm.dead_on_unwind, llvm.sret = !rec_HMem, llvm.writable}) -> ()
+ // CHECK-NEXT: %[[DEST:.*]] = cir.load %arg0 : !cir.ptr<!cir.ptr<!rec_HMem>>, !cir.ptr<!rec_HMem>
+ // CHECK-NEXT: cir.copy %[[SRET]] align(8) to %[[DEST]] align(8) : !cir.ptr<!rec_HMem>
+ // CHECK-NEXT: cir.return
+
+ cir.func @sret_nodom_smem(%arg0: !cir.ptr<!cir.ptr<!rec_SMem>>) {
+ %0 = cir.call @make_smem() : () -> !rec_SMem
+ %1 = cir.load %arg0 : !cir.ptr<!cir.ptr<!rec_SMem>>, !cir.ptr<!rec_SMem>
+ cir.store %0, %1 : !rec_SMem, !cir.ptr<!rec_SMem>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @sret_nodom_smem(%arg0: !cir.ptr<!cir.ptr<!rec_SMem>>)
+ // CHECK-NEXT: %[[SRET:.*]] = cir.alloca "sret" align(8) : !cir.ptr<!rec_SMem>
+ // CHECK-NEXT: cir.call @make_smem(%[[SRET]]) : (!cir.ptr<!rec_SMem> {llvm.align = 8 : i64, llvm.dead_on_unwind, llvm.sret = !rec_SMem, llvm.writable}) -> ()
+ // CHECK-NEXT: %[[DEST:.*]] = cir.load %arg0 : !cir.ptr<!cir.ptr<!rec_SMem>>, !cir.ptr<!rec_SMem>
+ // CHECK-NEXT: cir.copy %[[SRET]] align(8) to %[[DEST]] align(8) : !cir.ptr<!rec_SMem>
+ // CHECK-NEXT: cir.return
+
+ // A local is a complete object, so the copy into it takes the whole record.
+ cir.func @tp_store_local() {
+ %0 = cir.alloca "t" align(4) : !cir.ptr<!rec_TP>
+ %1 = cir.call @make_tp() : () -> !rec_TP
+ cir.store %1, %0 : !rec_TP, !cir.ptr<!rec_TP>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @tp_store_local()
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[T:.*]] = cir.alloca "t" align(4) : !cir.ptr<!rec_TP>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_tp() : () -> !u64i
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_TP>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %[[T]] align(4) : !cir.ptr<!rec_TP>
+ // CHECK-NEXT: cir.return
+
+ // Any other destination may be a subobject whose tail padding holds
+ // another object, so the copy stops at the data size.
+ cir.func @tp_store_ptr(%arg0: !cir.ptr<!rec_TP>) {
+ %0 = cir.call @make_tp() : () -> !rec_TP
+ cir.store %0, %arg0 : !rec_TP, !cir.ptr<!rec_TP>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @tp_store_ptr(%arg0: !cir.ptr<!rec_TP>)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_tp() : () -> !u64i
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_TP>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %arg0 align(4) skip_tail_padding : !cir.ptr<!rec_TP>
+ // CHECK-NEXT: cir.return
+
+ cir.func @sret_nodom_tpmem(%arg0: !cir.ptr<!cir.ptr<!rec_TPMem>>) {
+ %0 = cir.call @make_tpmem() : () -> !rec_TPMem
+ %1 = cir.load %arg0 : !cir.ptr<!cir.ptr<!rec_TPMem>>, !cir.ptr<!rec_TPMem>
+ cir.store %0, %1 : !rec_TPMem, !cir.ptr<!rec_TPMem>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @sret_nodom_tpmem(%arg0: !cir.ptr<!cir.ptr<!rec_TPMem>>)
+ // CHECK-NEXT: %[[SRET:.*]] = cir.alloca "sret" align(8) : !cir.ptr<!rec_TPMem>
+ // CHECK-NEXT: cir.call @make_tpmem(%[[SRET]]) : (!cir.ptr<!rec_TPMem> {llvm.align = 8 : i64, llvm.dead_on_unwind, llvm.sret = !rec_TPMem, llvm.writable}) -> ()
+ // CHECK-NEXT: %[[DEST:.*]] = cir.load %arg0 : !cir.ptr<!cir.ptr<!rec_TPMem>>, !cir.ptr<!rec_TPMem>
+ // CHECK-NEXT: cir.copy %[[SRET]] align(8) to %[[DEST]] align(8) skip_tail_padding : !cir.ptr<!rec_TPMem>
+ // CHECK-NEXT: cir.return
+
+ // A record with a layout gap but no pad member has no tail padding, so the
+ // copy through a pointer takes the whole record.
+ cir.func @sf_store_ptr(%arg0: !cir.ptr<!rec_SF>) {
+ %0 = cir.call @make_sf() : () -> !rec_SF
+ cir.store %0, %arg0 : !rec_SF, !cir.ptr<!rec_SF>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @sf_store_ptr(%arg0: !cir.ptr<!rec_SF>)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<![[SFPAIR]]>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_sf() : () -> ![[SFPAIR]]
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : ![[SFPAIR]], !cir.ptr<![[SFPAIR]]>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<![[SFPAIR]]> -> !cir.ptr<!rec_SF>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %arg0 align(8) : !cir.ptr<!rec_SF>
+ // CHECK-NEXT: cir.return
+
+ cir.func @tp_param_store_ptr(%arg0: !rec_TP, %arg1: !cir.ptr<!rec_TP>) {
+ cir.store %arg0, %arg1 : !rec_TP, !cir.ptr<!rec_TP>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @tp_param_store_ptr(%arg0: !u64i, %arg1: !cir.ptr<!rec_TP>)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.store %arg0, %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_TP>
+ // CHECK-NEXT: cir.copy %[[VIEW]] align(8) to %arg1 align(4) skip_tail_padding : !cir.ptr<!rec_TP>
+ // CHECK-NEXT: cir.return
+
+ // The copy that replaces a byval parameter's only store keeps the store's
+ // alignment.
+ cir.func @byval_tpmem_ptr(%arg0: !rec_TPMem, %arg1: !cir.ptr<!rec_TPMem>) {
+ cir.store align(1) %arg0, %arg1 : !rec_TPMem, !cir.ptr<!rec_TPMem>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @byval_tpmem_ptr(%arg0: !cir.ptr<!rec_TPMem> {llvm.align = 8 : i64, llvm.byval = !rec_TPMem, llvm.noundef}, %arg1: !cir.ptr<!rec_TPMem>)
+ // CHECK-NEXT: cir.copy %arg0 align(8) to %arg1 align(1) skip_tail_padding : !cir.ptr<!rec_TPMem>
+ // CHECK-NEXT: cir.return
+
+ // A byval parameter stored twice has no single spill, so each store is a
+ // copy from the incoming pointer.
+ cir.func @byval_twice_x87mem(%arg0: !rec_UX87Mem, %arg1: !cir.ptr<!rec_UX87Mem>) {
+ %0 = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87Mem>
+ cir.store %arg0, %0 : !rec_UX87Mem, !cir.ptr<!rec_UX87Mem>
+ cir.store %arg0, %arg1 : !rec_UX87Mem, !cir.ptr<!rec_UX87Mem>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @byval_twice_x87mem(%arg0: !cir.ptr<!rec_UX87Mem> {llvm.align = 16 : i64, llvm.byval = !rec_UX87Mem, llvm.noundef}, %arg1: !cir.ptr<!rec_UX87Mem>)
+ // CHECK-NEXT: %[[U:.*]] = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: cir.copy %arg0 align(16) to %[[U]] align(16) : !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: cir.copy %arg0 align(16) to %arg1 align(16) : !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: cir.return
+
+ // A byval parameter handed straight on is copied into the callee's slot.
+ cir.func @byval_forward_hmem(%arg0: !rec_HMem) {
+ cir.call @take_hmem(%arg0) : (!rec_HMem) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @byval_forward_hmem(%arg0: !cir.ptr<!rec_HMem> {llvm.align = 8 : i64, llvm.byval = !rec_HMem, llvm.noundef})
+ // CHECK-NEXT: %[[BYVAL:.*]] = cir.alloca "byval" align(8) : !cir.ptr<!rec_HMem>
+ // CHECK-NEXT: cir.copy %arg0 align(8) to %[[BYVAL]] align(8) : !cir.ptr<!rec_HMem>
+ // CHECK-NEXT: cir.call @take_hmem(%[[BYVAL]]) : (!cir.ptr<!rec_HMem> {llvm.align = 8 : i64, llvm.byval = !rec_HMem, llvm.noundef}) -> ()
+ // CHECK-NEXT: cir.return
+
+ // The consumer is rewritten first and stores the sret result into its
+ // byval slot. The producer's rewrite turns that store into a copy.
+ cir.func @byval_result_x87mem() {
+ %0 = cir.call @make_x87mem() : () -> !rec_UX87Mem
+ cir.call @take_x87mem(%0) : (!rec_UX87Mem) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @byval_result_x87mem()
+ // CHECK-NEXT: %[[SRET:.*]] = cir.alloca "sret" align(16) : !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: cir.call @make_x87mem(%[[SRET]])
+ // CHECK-NEXT: %[[BYVAL:.*]] = cir.alloca "byval" align(16) : !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: cir.copy %[[SRET]] align(16) to %[[BYVAL]] align(16) : !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: cir.call @take_x87mem(%[[BYVAL]]) : (!cir.ptr<!rec_UX87Mem> {llvm.align = 16 : i64, llvm.byval = !rec_UX87Mem, llvm.noundef}) -> ()
+ // CHECK-NEXT: cir.return
+
+ cir.func @byval_result_hmem() {
+ %0 = cir.call @make_hmem() : () -> !rec_HMem
+ cir.call @take_hmem(%0) : (!rec_HMem) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @byval_result_hmem()
+ // CHECK-NEXT: %[[SRET:.*]] = cir.alloca "sret" align(8) : !cir.ptr<!rec_HMem>
+ // CHECK-NEXT: cir.call @make_hmem(%[[SRET]])
+ // CHECK-NEXT: %[[BYVAL:.*]] = cir.alloca "byval" align(8) : !cir.ptr<!rec_HMem>
+ // CHECK-NEXT: cir.copy %[[SRET]] align(8) to %[[BYVAL]] align(8) : !cir.ptr<!rec_HMem>
+ // CHECK-NEXT: cir.call @take_hmem(%[[BYVAL]]) : (!cir.ptr<!rec_HMem> {llvm.align = 8 : i64, llvm.byval = !rec_HMem, llvm.noundef}) -> ()
+ // CHECK-NEXT: cir.return
+
+ // A register-class fetch copies out of the register save area through a
+ // temp, and the fetched value is copied into the local. The second fetch
+ // feeds a call rewritten before it, whose argument slot store becomes a
+ // copy.
+ cir.func @vaarg_x87(%arg0: !s32i, ...) {
+ %0 = cir.alloca "ap" align(16) : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>
+ %1 = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87>
+ %2 = cir.cast array_to_ptrdecay %0 : !cir.ptr<!cir.array<!rec___va_list_tag x 1>> -> !cir.ptr<!rec___va_list_tag>
+ %3 = cir.va_arg %2 : (!cir.ptr<!rec___va_list_tag>) -> !rec_UX87
+ cir.store align(16) %3, %1 : !rec_UX87, !cir.ptr<!rec_UX87>
+ %4 = cir.va_arg %2 : (!cir.ptr<!rec___va_list_tag>) -> !rec_UX87
+ cir.call @take_x87(%4) : (!rec_UX87) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @vaarg_x87(%arg0: !s32i, ...)
+ // CHECK-NEXT: %[[ASLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.alloca "ap"
+ // CHECK-NEXT: %[[U:.*]] = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87>
+ // CHECK: %[[TEMP:.*]] = cir.alloca "vaarg.reg" align(16) : !cir.ptr<!rec_UX87>
+ // CHECK: %[[REG:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.copy %[[REG]] align(8) to %[[TEMP]] align(16) : !cir.ptr<!rec_UX87>
+ // CHECK: %[[ADDR:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.copy %[[ADDR]] align(16) to %[[U]] align(16) : !cir.ptr<!rec_UX87>
+ // CHECK: %[[TEMP2:.*]] = cir.alloca "vaarg.reg" align(16) : !cir.ptr<!rec_UX87>
+ // CHECK: %[[REG2:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.copy %[[REG2]] align(8) to %[[TEMP2]] align(16) : !cir.ptr<!rec_UX87>
+ // CHECK: %[[ADDR2:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.copy %[[ADDR2]] align(16) to %[[ASLOT]] align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[AVIEW:.*]] = cir.cast bitcast %[[ASLOT]] : !cir.ptr<!rec_UX87> -> !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[AVIEW]][0] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[LO:.*]] = cir.load %[[LOP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[AVIEW]][1] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[HI:.*]] = cir.load %[[HIP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_x87(%[[LO]], %[[HI]]) : (!u64i, !u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ cir.func @vaarg_h(%arg0: !s32i, ...) {
+ %0 = cir.alloca "ap" align(16) : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>
+ %1 = cir.alloca "h" align(4) : !cir.ptr<!rec_H>
+ %2 = cir.cast array_to_ptrdecay %0 : !cir.ptr<!cir.array<!rec___va_list_tag x 1>> -> !cir.ptr<!rec___va_list_tag>
+ %3 = cir.va_arg %2 : (!cir.ptr<!rec___va_list_tag>) -> !rec_H
+ cir.store %3, %1 : !rec_H, !cir.ptr<!rec_H>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @vaarg_h(%arg0: !s32i, ...)
+ // CHECK-NEXT: cir.alloca "ap"
+ // CHECK-NEXT: %[[H:.*]] = cir.alloca "h" align(4) : !cir.ptr<!rec_H>
+ // CHECK: %[[ADDR:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.copy %[[ADDR]] align(4) to %[[H]] align(4) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.return
+
+ // A memory-class fetch reads the overflow area. The second one feeds a
+ // byval slot, filled by a copy.
+ cir.func @vaarg_x87mem(%arg0: !s32i, ...) {
+ %0 = cir.alloca "ap" align(16) : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>
+ %1 = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87Mem>
+ %2 = cir.cast array_to_ptrdecay %0 : !cir.ptr<!cir.array<!rec___va_list_tag x 1>> -> !cir.ptr<!rec___va_list_tag>
+ %3 = cir.va_arg %2 : (!cir.ptr<!rec___va_list_tag>) -> !rec_UX87Mem
+ cir.store align(16) %3, %1 : !rec_UX87Mem, !cir.ptr<!rec_UX87Mem>
+ %4 = cir.va_arg %2 : (!cir.ptr<!rec___va_list_tag>) -> !rec_UX87Mem
+ cir.call @take_x87mem(%4) : (!rec_UX87Mem) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @vaarg_x87mem(%arg0: !s32i, ...)
+ // CHECK-NEXT: cir.alloca "ap"
+ // CHECK-NEXT: %[[U:.*]] = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87Mem>
+ // CHECK-NOT: cir.ternary
+ // CHECK: %[[ADDR:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: cir.copy %[[ADDR]] align(16) to %[[U]] align(16) : !cir.ptr<!rec_UX87Mem>
+ // CHECK: %[[ADDR2:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!u8i> -> !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: %[[BYVAL:.*]] = cir.alloca "byval" align(16) : !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: cir.copy %[[ADDR2]] align(16) to %[[BYVAL]] align(16) : !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: cir.call @take_x87mem(%[[BYVAL]])
+ // CHECK-NEXT: cir.return
+
+ // The producer is rewritten before the consumer, so the consumer reads the
+ // producer's slot in place.
+ cir.func @param_forward_late_x87(%arg0: !rec_UX87) {
+ cir.call @take_late_x87(%arg0) : (!rec_UX87) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @param_forward_late_x87(%arg0: !u64i, %arg1: !u64i)
+ // CHECK: %[[VIEW:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[AVIEW:.*]] = cir.cast bitcast %[[VIEW]] : !cir.ptr<!rec_UX87> -> !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[AVIEW]][0] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[LO:.*]] = cir.load align(16) %[[LOP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[AVIEW]][1] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[HI:.*]] = cir.load align(8) %[[HIP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_late_x87(%[[LO]], %[[HI]]) : (!u64i, !u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ cir.func @param_forward_late_h(%arg0: !rec_H) {
+ cir.call @take_late_h(%arg0) : (!rec_H) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @param_forward_late_h(%arg0: !u64i)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: cir.store %arg0, %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[AVIEW:.*]] = cir.cast bitcast %[[VIEW]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[V:.*]] = cir.load align(8) %[[AVIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_late_h(%[[V]]) : (!u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ cir.func @result_feed_late_x87() {
+ %0 = cir.call @make_x87() : () -> !rec_UX87
+ cir.call @take_late_x87(%0) : (!rec_UX87) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @result_feed_late_x87()
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_x87() : () -> ![[PAIR]]
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : ![[PAIR]], !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[AVIEW:.*]] = cir.cast bitcast %[[VIEW]] : !cir.ptr<!rec_UX87> -> !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[AVIEW]][0] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[LO:.*]] = cir.load align(16) %[[LOP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[AVIEW]][1] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[HI:.*]] = cir.load align(8) %[[HIP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_late_x87(%[[LO]], %[[HI]]) : (!u64i, !u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ cir.func @result_feed_late_h() {
+ %0 = cir.call @make_h() : () -> !rec_H
+ cir.call @take_late_h(%0) : (!rec_H) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @result_feed_late_h()
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_h() : () -> !u64i
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[AVIEW:.*]] = cir.cast bitcast %[[VIEW]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[V:.*]] = cir.load align(8) %[[AVIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_late_h(%[[V]]) : (!u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ // The sret slot is handed on as the byval argument.
+ cir.func @byval_result_late_x87mem() {
+ %0 = cir.call @make_x87mem() : () -> !rec_UX87Mem
+ cir.call @take_late_x87mem(%0) : (!rec_UX87Mem) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @byval_result_late_x87mem()
+ // CHECK-NEXT: %[[SRET:.*]] = cir.alloca "sret" align(16) : !cir.ptr<!rec_UX87Mem>
+ // CHECK-NEXT: cir.call @make_x87mem(%[[SRET]])
+ // CHECK-NEXT: cir.call @take_late_x87mem(%[[SRET]]) : (!cir.ptr<!rec_UX87Mem> {llvm.align = 16 : i64, llvm.byval = !rec_UX87Mem, llvm.noundef}) -> ()
+ // CHECK-NEXT: cir.return
+
+ // The definition is rewritten before the producer, so the return stores
+ // the result into a slot and the producer's rewrite turns that store into
+ // a copy.
+ cir.func @result_return_late_x87() -> !rec_UX87 {
+ %0 = cir.call @make_late_x87() : () -> !rec_UX87
+ cir.return %0 : !rec_UX87
+ }
+
+ cir.func @result_return_late_h() -> !rec_H {
+ %0 = cir.call @make_late_h() : () -> !rec_H
+ cir.return %0 : !rec_H
+ }
+
+ cir.func private @take_late_x87(!rec_UX87)
+ cir.func private @take_late_h(!rec_H)
+ cir.func private @take_late_x87mem(!rec_UX87Mem)
+ cir.func private @make_late_x87() -> !rec_UX87
+ cir.func private @make_late_h() -> !rec_H
+
+ // CHECK-LABEL: cir.func @result_return_late_x87()
+ // CHECK-SAME: -> ![[PAIR]]
+ // CHECK-NEXT: %[[RSLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_late_x87() : () -> ![[PAIR]]
+ // CHECK-NEXT: cir.store %[[RES]], %[[RSLOT]] : ![[PAIR]], !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[RVIEW:.*]] = cir.cast bitcast %[[RSLOT]] : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: cir.copy %[[RVIEW]] align(16) to %[[SLOT]] align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_UX87> -> !cir.ptr<![[PAIR]]>
+ // CHECK-NEXT: %[[RET:.*]] = cir.load %[[VIEW]] : !cir.ptr<![[PAIR]]>, ![[PAIR]]
+ // CHECK-NEXT: cir.return %[[RET]] : ![[PAIR]]
+
+ // CHECK-LABEL: cir.func @result_return_late_h() -> !u64i
+ // CHECK-NEXT: %[[RSLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_late_h() : () -> !u64i
+ // CHECK-NEXT: cir.store %[[RES]], %[[RSLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[RVIEW:.*]] = cir.cast bitcast %[[RSLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_H>
+ // CHECK-NEXT: cir.copy %[[RVIEW]] align(8) to %[[SLOT]] align(4) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[RET:.*]] = cir.load %[[VIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.return %[[RET]] : !u64i
+
+ // CHECK: cir.func private @take_late_x87(!u64i, !u64i)
+ // CHECK: cir.func private @take_late_h(!u64i)
+ // CHECK: cir.func private @take_late_x87mem(!cir.ptr<!rec_UX87Mem> {llvm.align = 16 : i64, llvm.byval = !rec_UX87Mem, llvm.noundef})
+ // CHECK: cir.func private @make_late_x87() -> ![[PAIR]]
+ // CHECK: cir.func private @make_late_h() -> !u64i
+
+ // A constant carries no bytes beyond its own value, so it is still stored
+ // into the argument slot by value.
+ cir.func @const_feed_x87() {
+ %0 = cir.const #cir.zero : !rec_UX87
+ cir.call @take_x87(%0) : (!rec_UX87) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @const_feed_x87()
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[C:.*]] = cir.const #cir.zero : !rec_UX87
+ // CHECK-NEXT: cir.store %[[C]], %[[SLOT]] : !rec_UX87, !cir.ptr<!rec_UX87>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_UX87> -> !cir.ptr<![[PAIR]]>
+ // CHECK: cir.call @take_x87(%{{.*}}, %{{.*}}) : (!u64i, !u64i) -> ()
+ // MOVE: cir.store %{{.*}}, %{{.*}} : !rec_UX87, !cir.ptr<!rec_UX87>
+
+ cir.func @const_feed_h() {
+ %0 = cir.const #cir.zero : !rec_H
+ cir.call @take_h(%0) : (!rec_H) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @const_feed_h()
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[C:.*]] = cir.const #cir.zero : !rec_H
+ // CHECK-NEXT: cir.store %[[C]], %[[SLOT]] : !rec_H, !cir.ptr<!rec_H>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_H> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[V:.*]] = cir.load %[[VIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_h(%[[V]]) : (!u64i) -> ()
+ // MOVE-LABEL: cir.func @const_feed_h()
+ // MOVE: cir.store %{{.*}}, %{{.*}} : !rec_H, !cir.ptr<!rec_H>
+
+ cir.func @const_feed_hmem() {
+ %0 = cir.const #cir.zero : !rec_HMem
+ cir.call @take_hmem(%0) : (!rec_HMem) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @const_feed_hmem()
+ // CHECK-NEXT: %[[C:.*]] = cir.const #cir.zero : !rec_HMem
+ // CHECK-NEXT: %[[BYVAL:.*]] = cir.alloca "byval" align(8) : !cir.ptr<!rec_HMem>
+ // CHECK-NEXT: cir.store %[[C]], %[[BYVAL]] : !rec_HMem, !cir.ptr<!rec_HMem>
+ // CHECK-NEXT: cir.call @take_hmem(%[[BYVAL]])
+ // MOVE-LABEL: cir.func @const_feed_hmem()
+ // MOVE: cir.store %{{.*}}, %{{.*}} : !rec_HMem, !cir.ptr<!rec_HMem>
+
+ // A member read reads the record back out of the result slot.
+ cir.func @result_extract_s() -> !s32i {
+ %0 = cir.call @make_s() : () -> !rec_S
+ %1 = cir.extract_member %0[1] : !rec_S -> !s32i
+ cir.return %1 : !s32i
+ }
+
+ // CHECK-LABEL: cir.func @result_extract_s() -> !s32i
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[RES:.*]] = cir.call @make_s() : () -> !u64i
+ // CHECK-NEXT: cir.store %[[RES]], %[[SLOT]] : !u64i, !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_S>
+ // CHECK-NEXT: %[[REC:.*]] = cir.load align(8) %[[VIEW]] : !cir.ptr<!rec_S>, !rec_S
+ // CHECK-NEXT: %[[FIELD:.*]] = cir.extract_member %[[REC]][1] : !rec_S -> !s32i
+ // CHECK-NEXT: cir.return %[[FIELD]] : !s32i
+ // MOVE-LABEL: cir.func @result_extract_s()
+ // MOVE: cir.load align(8) %{{.*}} : !cir.ptr<!rec_S>, !rec_S
+}
+
+// LLVM-LABEL: define void @param_spill_x87(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,]+}}, ptr align 16 %{{[^,]+}}, i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @param_spill_h(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 8, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @param_spill_s(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @param_spill_sf(i64 %{{[^,)]+}}, i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @param_unused_x87(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @param_unused_h(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @param_unused_s(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @param_forward_x87(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @take_x87(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-LABEL: define void @param_forward_h(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @take_h(i64 %{{[^,)]+}})
+// LLVM-LABEL: define void @param_forward_sf(i64 %{{[^,)]+}}, i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @take_sf(i64 %{{[^,)]+}}, i32 %{{[^,)]+}})
+// LLVM-LABEL: define { i64, i64 } @param_return_x87(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: load { i64, i64 }, ptr %{{[^,]+}}, align 16
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define i64 @param_return_h(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define { i64, i32 } @param_return_sf(i64 %{{[^,)]+}}, i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define { i64, i64 } @ret_local_x87()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define i64 @ret_local_h()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @call_inplace_x87()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: load i64, ptr %{{[^,]+}}, align 16
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: load i64, ptr %{{[^,]+}}, align 8
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @call_inplace_h()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: load i64, ptr %{{[^,]+}}, align 4
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @call_inplace_s()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @call_inplace_sf()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @call_inplace_s16()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: %[[LO:[0-9]+]] = load i64, ptr %{{[^,]+}}, align 4
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: %[[HI:[0-9]+]] = load i64, ptr %{{[^,]+}}, align 4
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @take_s16(i64 %[[LO]], i64 %[[HI]])
+// LLVM-LABEL: define void @call_copied_x87(
+// LLVM-SAME: ptr %[[SRC:[^,)]+]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,]+}}, ptr align 16 %[[SRC]], i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @call_copied_h(
+// LLVM-SAME: ptr %[[SRC:[^,)]+]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{[^,]+}}, ptr align 4 %[[SRC]], i64 8, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @call_small_u3()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %{{[^,]+}}, ptr align 1 %{{[^,]+}}, i64 3, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: load i24, ptr %{{[^,]+}}, align 4
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @call_small_s12()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{[^,]+}}, ptr align 4 %{{[^,]+}}, i64 12, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @result_store_x87()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,]+}}, ptr align 16 %{{[^,]+}}, i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @result_store_h()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 8, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @result_store_s()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @result_store_sf()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @result_stride_x87(ptr %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: %[[ELT:[0-9]+]] = getelementptr %union.UX87, ptr %{{[^,]+}}, i64 3
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[ELT]], ptr align 16 %{{[^,]+}}, i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @result_stride_h(ptr %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: %[[ELT:[0-9]+]] = getelementptr %union.H, ptr %{{[^,]+}}, i64 3
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[ELT]], ptr align 8 %{{[^,]+}}, i64 8, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @result_feed_x87()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @result_feed_h()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define { i64, i64 } @result_return_x87()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define i64 @result_return_h()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @result_volatile_x87(
+// LLVM-SAME: ptr %[[DST:[^,)]+]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[DST]], ptr align 16 %{{[^,]+}}, i64 16, i1 true)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @result_volatile_h(
+// LLVM-SAME: ptr %[[DST:[^,)]+]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[DST]], ptr align 8 %{{[^,]+}}, i64 8, i1 true)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @sret_nodom_x87mem(ptr %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,]+}}, ptr align 16 %{{[^,]+}}, i64 32, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @sret_nodom_hmem(ptr %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 32, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @sret_nodom_smem(ptr %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 24, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @tp_store_local()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 8, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @tp_store_ptr(ptr %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 5, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @sret_nodom_tpmem(ptr %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 25, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @sf_store_ptr(ptr %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @tp_param_store_ptr(i64 %{{[^,)]+}}, ptr %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 5, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @byval_tpmem_ptr(ptr noundef byval(%struct.TPMem) align 8 %{{[^,)]+}}, ptr %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 1 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 25, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @byval_twice_x87mem(
+// LLVM-SAME: ptr noundef byval(%union.UX87Mem) align 16 %[[SRC:[^,)]+]], ptr %[[DST:[^,)]+]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,]+}}, ptr align 16 %[[SRC]], i64 32, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[DST]], ptr align 16 %[[SRC]], i64 32, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @byval_forward_hmem(ptr noundef byval(%union.HMem) align 8 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @byval_result_x87mem()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @byval_result_hmem()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @vaarg_x87(i32 %{{[^,)]+}}, ...)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,]+}}, ptr align 8 %{{[^,]+}}, i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @vaarg_h(i32 %{{[^,)]+}}, ...)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %{{[^,]+}}, ptr align 4 %{{[^,]+}}, i64 8, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @vaarg_x87mem(i32 %{{[^,)]+}}, ...)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @param_forward_late_x87(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @param_forward_late_h(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @result_feed_late_x87()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @result_feed_late_h()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @byval_result_late_x87mem()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM: call void @make_x87mem(ptr dead_on_unwind writable sret(%union.UX87Mem) align 16 %[[SRET:[^,)]+]])
+// LLVM-NEXT: call void @take_late_x87mem(ptr noundef byval(%union.UX87Mem) align 16 %[[SRET]])
+// LLVM-LABEL: define { i64, i64 } @result_return_late_x87()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define i64 @result_return_late_h()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* (x86_fp80|<?\{ ?x86_fp80|%union\.|%struct\.)}}
+// LLVM-LABEL: define void @const_feed_x87()
+// LLVM: call void @take_x87(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-LABEL: define void @const_feed_h()
+// LLVM: call void @take_h(i64 %{{[^,)]+}})
+// LLVM-LABEL: define void @const_feed_hmem()
+// LLVM: call void @take_hmem(ptr noundef byval(%union.HMem) align 8 %{{[^,)]+}})
+// LLVM-LABEL: define i32 @result_extract_s()
+
+// -----
+
+!s8i = !cir.int<s, 8>
+!s32i = !cir.int<s, 32>
+!rec_HS = !cir.struct<"HS" {data !s8i, data !s32i}>
+!rec_H = !cir.union<"H" {data !rec_HS, data !cir.array<!s32i x 2>}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func @atomic_store_param_h(%arg0: !rec_H, %arg1: !cir.ptr<!rec_H>) {
+ // expected-error @below {{atomic store of a record argument or result not yet implemented in CallConvLowering}}
+ cir.store atomic(seq_cst) %arg0, %arg1 : !rec_H, !cir.ptr<!rec_H>
+ cir.return
+ }
+}
+
+// -----
+
+!u8i = !cir.int<u, 8>
+!rec_UX87 = !cir.union<"UX87" {data !cir.long_double<!cir.f80>, data !cir.array<!u8i x 16>}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func @atomic_store_flat_param_x87(%arg0: !rec_UX87, %arg1: !cir.ptr<!rec_UX87>) {
+ // expected-error @below {{atomic store of a record argument or result not yet implemented in CallConvLowering}}
+ cir.store atomic(seq_cst) %arg0, %arg1 : !rec_UX87, !cir.ptr<!rec_UX87>
+ cir.return
+ }
+}
+
+// -----
+
+!u8i = !cir.int<u, 8>
+!rec_UX87Mem = !cir.union<"UX87Mem" {data !cir.complex<!cir.long_double<!cir.f80>>, data !cir.array<!u8i x 32>}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func @atomic_store_byval_param(%arg0: !rec_UX87Mem, %arg1: !cir.ptr<!rec_UX87Mem>) {
+ %0 = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87Mem>
+ cir.store %arg0, %0 : !rec_UX87Mem, !cir.ptr<!rec_UX87Mem>
+ // expected-error @below {{atomic store of a record argument or result not yet implemented in CallConvLowering}}
+ cir.store atomic(seq_cst) %arg0, %arg1 : !rec_UX87Mem, !cir.ptr<!rec_UX87Mem>
+ cir.return
+ }
+}
+
+// -----
+
+!u8i = !cir.int<u, 8>
+!rec_UX87 = !cir.union<"UX87" {data !cir.long_double<!cir.f80>, data !cir.array<!u8i x 16>}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func private @make_x87() -> !rec_UX87
+ cir.func @atomic_store_result_x87(%arg0: !cir.ptr<!rec_UX87>) {
+ %0 = cir.call @make_x87() : () -> !rec_UX87
+ // expected-error @below {{atomic store of a record argument or result not yet implemented in CallConvLowering}}
+ cir.store atomic(seq_cst) %0, %arg0 : !rec_UX87, !cir.ptr<!rec_UX87>
+ cir.return
+ }
+}
+
+// -----
+
+!u8i = !cir.int<u, 8>
+!rec_UX87Mem = !cir.union<"UX87Mem" {data !cir.complex<!cir.long_double<!cir.f80>>, data !cir.array<!u8i x 32>}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func private @make_x87mem() -> !rec_UX87Mem
+ cir.func @atomic_store_sret_x87mem(%arg0: !cir.ptr<!cir.ptr<!rec_UX87Mem>>) {
+ %0 = cir.call @make_x87mem() : () -> !rec_UX87Mem
+ %1 = cir.load %arg0 : !cir.ptr<!cir.ptr<!rec_UX87Mem>>, !cir.ptr<!rec_UX87Mem>
+ // expected-error @below {{atomic store of a record argument or result not yet implemented in CallConvLowering}}
+ cir.store atomic(seq_cst) %0, %1 : !rec_UX87Mem, !cir.ptr<!rec_UX87Mem>
+ cir.return
+ }
+}
+
+// -----
+
+!s8i = !cir.int<s, 8>
+!s32i = !cir.int<s, 32>
+!u32i = !cir.int<u, 32>
+!void = !cir.void
+!rec_HS = !cir.struct<"HS" {data !s8i, data !s32i}>
+!rec_H = !cir.union<"H" {data !rec_HS, data !cir.array<!s32i x 2>}>
+!rec___va_list_tag = !cir.struct<"__va_list_tag" {data !u32i, data !u32i, data !cir.ptr<!void>, data !cir.ptr<!void>}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func @atomic_store_vaarg_h(%arg0: !cir.ptr<!rec___va_list_tag>, %arg1: !cir.ptr<!rec_H>) {
+ %0 = cir.va_arg %arg0 : (!cir.ptr<!rec___va_list_tag>) -> !rec_H
+ // expected-error @below {{atomic store of a record argument or result not yet implemented in CallConvLowering}}
+ cir.store atomic(seq_cst) %0, %arg1 : !rec_H, !cir.ptr<!rec_H>
+ cir.return
+ }
+}
+
+// -----
+
+!s8i = !cir.int<s, 8>
+!s32i = !cir.int<s, 32>
+!rec_HS = !cir.struct<"HS" {data !s8i, data !s32i}>
+!rec_H = !cir.union<"H" {data !rec_HS, data !cir.array<!s32i x 2>}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func private @make_h() -> !rec_H
+
+ // A call result consumed by anything but a store, call, return or member
+ // read would be loaded from its slot as a value.
+ cir.func @result_yield_h(%arg0: !cir.bool, %arg1: !cir.ptr<!rec_H>) {
+ %0 = cir.call @make_h() : () -> !rec_H
+ %1 = cir.ternary(%arg0, true {
+ // expected-error @below {{use of a record argument or result other than by a store, call, return or member read not yet implemented in CallConvLowering}}
+ cir.yield %0 : !rec_H
+ }, false {
+ %2 = cir.load %arg1 : !cir.ptr<!rec_H>, !rec_H
+ cir.yield %2 : !rec_H
+ }) : (!cir.bool) -> !rec_H
+ cir.store %1, %arg1 : !rec_H, !cir.ptr<!rec_H>
+ cir.return
+ }
+}
diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-record-load-nyi.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-record-load-nyi.cir
new file mode 100644
index 00000000000000..94586dd26f4519
--- /dev/null
+++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-record-load-nyi.cir
@@ -0,0 +1,121 @@
+// RUN: cir-opt %s -split-input-file -verify-diagnostics -cir-call-conv-lowering=target=x86_64 -o %t.cir
+
+!s32i = !cir.int<s, 32>
+!rec_S = !cir.struct<"S" {data !s32i, data !s32i}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func private @take_s(!rec_S)
+ cir.func @direct_atomic(%arg0: !cir.ptr<!rec_S>) {
+ %0 = cir.load atomic(seq_cst) %arg0 : !cir.ptr<!rec_S>, !rec_S
+ // expected-error @below {{atomic load of a record operand not yet implemented in CallConvLowering}}
+ cir.call @take_s(%0) : (!rec_S) -> ()
+ cir.return
+ }
+}
+
+// -----
+
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+!rec_SF = !cir.struct<"SF" {data !s64i, data !s32i}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func private @take_sf(!rec_SF)
+ cir.func @flattened_atomic(%arg0: !cir.ptr<!rec_SF>) {
+ %0 = cir.load atomic(seq_cst) %arg0 : !cir.ptr<!rec_SF>, !rec_SF
+ // expected-error @below {{atomic load of a record operand not yet implemented in CallConvLowering}}
+ cir.call @take_sf(%0) : (!rec_SF) -> ()
+ cir.return
+ }
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_SMem = !cir.struct<"SMem" {data !s64i, data !s64i, data !s64i}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func private @take_smem(!rec_SMem)
+ cir.func @byval_atomic(%arg0: !cir.ptr<!rec_SMem>) {
+ %0 = cir.load atomic(seq_cst) %arg0 : !cir.ptr<!rec_SMem>, !rec_SMem
+ // expected-error @below {{atomic load of a record operand not yet implemented in CallConvLowering}}
+ cir.call @take_smem(%0) : (!rec_SMem) -> ()
+ cir.return
+ }
+}
+
+// -----
+
+!s32i = !cir.int<s, 32>
+!rec_S = !cir.struct<"S" {data !s32i, data !s32i}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func @return_atomic(%arg0: !cir.ptr<!rec_S>) -> !rec_S {
+ %0 = cir.load atomic(seq_cst) %arg0 : !cir.ptr<!rec_S>, !rec_S
+ // expected-error @below {{atomic load of a record operand not yet implemented in CallConvLowering}}
+ cir.return %0 : !rec_S
+ }
+}
+
+// -----
+
+!s64i = !cir.int<s, 64>
+!rec_SMem = !cir.struct<"SMem" {data !s64i, data !s64i, data !s64i}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func @byval_atomic_spill(%arg0: !rec_SMem) {
+ %0 = cir.alloca "s" align(8) : !cir.ptr<!rec_SMem>
+ // expected-error @below {{atomic store of a record argument or result not yet implemented in CallConvLowering}}
+ cir.store atomic(seq_cst) %arg0, %0 : !rec_SMem, !cir.ptr<!rec_SMem>
+ cir.return
+ }
+}
+
+// -----
+
+!s8i = !cir.int<s, 8>
+!s32i = !cir.int<s, 32>
+!rec_HS = !cir.struct<"HS" {data !s8i, data !s32i}>
+!rec_H = !cir.union<"H" {data !rec_HS, data !cir.array<!s32i x 2>}>
+!rec_W = !cir.struct<"W" {data !rec_H}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func private @make_w() -> !rec_W
+ cir.func @result_record_member(%arg0: !cir.ptr<!rec_H>) {
+ %0 = cir.call @make_w() : () -> !rec_W
+ // expected-error @below {{use of a record argument or result other than by a store, call, return or member read not yet implemented in CallConvLowering}}
+ %1 = cir.extract_member %0[0] : !rec_W -> !rec_H
+ cir.store %1, %arg0 : !rec_H, !cir.ptr<!rec_H>
+ cir.return
+ }
+}
diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-record-load-volatile.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-record-load-volatile.cir
new file mode 100644
index 00000000000000..05ce78b9a700e9
--- /dev/null
+++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-record-load-volatile.cir
@@ -0,0 +1,172 @@
+// RUN: cir-opt %s -cir-call-conv-lowering=target=x86_64 | FileCheck %s
+// RUN: cir-opt %s -cir-call-conv-lowering=target=x86_64 -cir-to-llvm -o - \
+// RUN: | cir-translate -mlir-to-llvmir --allow-unregistered-dialect \
+// RUN: | FileCheck %s --check-prefix=LLVM
+
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+!rec_S = !cir.struct<"S" {data !s32i, data !s32i}>
+!rec_SF = !cir.struct<"SF" {data !s64i, data !s32i}>
+!rec_SMem = !cir.struct<"SMem" {data !s64i, data !s64i, data !s64i}>
+
+module attributes {
+ cir.triple = "x86_64-unknown-linux-gnu",
+ dlti.dl_spec = #dlti.dl_spec<
+ #dlti.dl_entry<i32, dense<32>: vector<2xi64>>,
+ #dlti.dl_entry<i64, dense<64>: vector<2xi64>>>
+} {
+ cir.func private @take_s(!rec_S)
+ cir.func private @take_sf(!rec_SF)
+ cir.func private @take_smem(!rec_SMem)
+
+ cir.func @direct_volatile(%arg0: !cir.ptr<!rec_S>) {
+ %0 = cir.load volatile align(4) %arg0 : !cir.ptr<!rec_S>, !rec_S
+ cir.call @take_s(%0) : (!rec_S) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @direct_volatile(%arg0: !cir.ptr<!rec_S>)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_S>
+ // CHECK-NEXT: cir.copy %arg0 align(4) to %[[SLOT]] align(8) volatile : !cir.ptr<!rec_S>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_S> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VAL:.*]] = cir.load align(8) %[[VIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_s(%[[VAL]]) : (!u64i) -> ()
+ // CHECK-NEXT: cir.return
+
+ // LLVM-LABEL: define void @direct_volatile(ptr %0)
+ // LLVM-NEXT: %[[SLOT:.*]] = alloca %struct.S, align 8
+ // LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[SLOT]], ptr align 4 %0, i64 8, i1 true)
+ // LLVM-NEXT: %[[VAL:.*]] = load i64, ptr %[[SLOT]], align 8
+ // LLVM-NEXT: call void @take_s(i64 %[[VAL]])
+
+ // An alloca is copied too, never read in place.
+ cir.func @local_direct_volatile() {
+ %0 = cir.alloca "s" align(4) : !cir.ptr<!rec_S>
+ %1 = cir.load volatile %0 : !cir.ptr<!rec_S>, !rec_S
+ cir.call @take_s(%1) : (!rec_S) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @local_direct_volatile()
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_S>
+ // CHECK-NEXT: %[[S:.*]] = cir.alloca "s" align(4) : !cir.ptr<!rec_S>
+ // CHECK-NEXT: cir.copy %[[S]] align(4) to %[[SLOT]] align(8) volatile : !cir.ptr<!rec_S>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_S> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VAL:.*]] = cir.load align(8) %[[VIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.call @take_s(%[[VAL]]) : (!u64i) -> ()
+
+ // LLVM-LABEL: define void @local_direct_volatile()
+ // LLVM-NEXT: %[[SLOT:.*]] = alloca %struct.S, align 8
+ // LLVM-NEXT: %[[S:.*]] = alloca %struct.S, align 4
+ // LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[SLOT]], ptr align 4 %[[S]], i64 8, i1 true)
+ // LLVM-NEXT: %[[VAL:.*]] = load i64, ptr %[[SLOT]], align 8
+
+ cir.func @flattened_volatile(%arg0: !cir.ptr<!rec_SF>) {
+ %0 = cir.load volatile align(8) %arg0 : !cir.ptr<!rec_SF>, !rec_SF
+ cir.call @take_sf(%0) : (!rec_SF) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @flattened_volatile(%arg0: !cir.ptr<!rec_SF>)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_SF>
+ // CHECK-NEXT: cir.copy %arg0 align(8) to %[[SLOT]] align(8) volatile : !cir.ptr<!rec_SF>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_SF> -> !cir.ptr<![[PAIR:rec_anon_struct[0-9]*]]>
+ // CHECK-NEXT: %[[LOP:.*]] = cir.get_member %[[VIEW]][0] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!s64i>
+ // CHECK-NEXT: %[[LO:.*]] = cir.load %[[LOP]] : !cir.ptr<!s64i>, !s64i
+ // CHECK-NEXT: %[[HIP:.*]] = cir.get_member %[[VIEW]][1] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!s32i>
+ // CHECK-NEXT: %[[HI:.*]] = cir.load %[[HIP]] : !cir.ptr<!s32i>, !s32i
+ // CHECK-NEXT: cir.call @take_sf(%[[LO]], %[[HI]]) : (!s64i, !s32i) -> ()
+
+ // LLVM-LABEL: define void @flattened_volatile(ptr %0)
+ // LLVM-NEXT: %[[SLOT:.*]] = alloca %struct.SF, align 8
+ // LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[SLOT]], ptr align 8 %0, i64 16, i1 true)
+
+ // The flattened read of a volatile load is copied even from an alloca.
+ cir.func @local_flattened_volatile() {
+ %0 = cir.alloca "s" align(8) : !cir.ptr<!rec_SF>
+ %1 = cir.load volatile %0 : !cir.ptr<!rec_SF>, !rec_SF
+ cir.call @take_sf(%1) : (!rec_SF) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @local_flattened_volatile()
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_SF>
+ // CHECK-NEXT: %[[S:.*]] = cir.alloca "s" align(8) : !cir.ptr<!rec_SF>
+ // CHECK-NEXT: cir.copy %[[S]] align(8) to %[[SLOT]] align(8) volatile : !cir.ptr<!rec_SF>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_SF> -> !cir.ptr<![[PAIR]]>
+
+ // LLVM-LABEL: define void @local_flattened_volatile()
+ // LLVM-NEXT: %[[SLOT:.*]] = alloca %struct.SF, align 8
+ // LLVM-NEXT: %[[S:.*]] = alloca %struct.SF, align 8
+ // LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[SLOT]], ptr align 8 %[[S]], i64 16, i1 true)
+
+ // A byval parameter's volatile spill is a volatile copy.
+ cir.func @byval_volatile_spill(%arg0: !rec_SMem) {
+ %0 = cir.alloca "s" align(8) : !cir.ptr<!rec_SMem>
+ cir.store volatile %arg0, %0 : !rec_SMem, !cir.ptr<!rec_SMem>
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @byval_volatile_spill(%arg0: !cir.ptr<!rec_SMem> {llvm.align = 8 : i64, llvm.byval = !rec_SMem, llvm.noundef})
+ // CHECK-NEXT: %[[S:.*]] = cir.alloca "s" align(8) : !cir.ptr<!rec_SMem>
+ // CHECK-NEXT: cir.copy %arg0 align(8) to %[[S]] align(8) volatile : !cir.ptr<!rec_SMem>
+ // CHECK-NEXT: cir.return
+
+ // LLVM-LABEL: define void @byval_volatile_spill(ptr noundef byval(%struct.SMem) align 8 %0)
+ // LLVM-NEXT: %[[S:.*]] = alloca %struct.SMem, align 8
+ // LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[S]], ptr align 8 %0, i64 24, i1 true)
+
+ cir.func @byval_volatile(%arg0: !cir.ptr<!rec_SMem>) {
+ %0 = cir.load volatile align(8) %arg0 : !cir.ptr<!rec_SMem>, !rec_SMem
+ cir.call @take_smem(%0) : (!rec_SMem) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @byval_volatile(%arg0: !cir.ptr<!rec_SMem>)
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "byval" align(8) : !cir.ptr<!rec_SMem>
+ // CHECK-NEXT: cir.copy %arg0 align(8) to %[[SLOT]] align(8) volatile : !cir.ptr<!rec_SMem>
+ // CHECK-NEXT: cir.call @take_smem(%[[SLOT]])
+
+ // LLVM-LABEL: define void @byval_volatile(ptr %0)
+ // LLVM-NEXT: %[[SLOT:.*]] = alloca %struct.SMem, align 8
+ // LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[SLOT]], ptr align 8 %0, i64 24, i1 true)
+ // LLVM-NEXT: call void @take_smem(ptr noundef byval(%struct.SMem) align 8 %[[SLOT]])
+
+ // A volatile load is not forwarded, even from an alloca nothing writes.
+ cir.func @local_byval_volatile() {
+ %0 = cir.alloca "s" align(8) : !cir.ptr<!rec_SMem>
+ %1 = cir.load volatile %0 : !cir.ptr<!rec_SMem>, !rec_SMem
+ cir.call @take_smem(%1) : (!rec_SMem) -> ()
+ cir.return
+ }
+
+ // CHECK-LABEL: cir.func @local_byval_volatile()
+ // CHECK-NEXT: %[[S:.*]] = cir.alloca "s" align(8) : !cir.ptr<!rec_SMem>
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "byval" align(8) : !cir.ptr<!rec_SMem>
+ // CHECK-NEXT: cir.copy %[[S]] align(8) to %[[SLOT]] align(8) volatile : !cir.ptr<!rec_SMem>
+ // CHECK-NEXT: cir.call @take_smem(%[[SLOT]])
+
+ // LLVM-LABEL: define void @local_byval_volatile()
+ // LLVM-NEXT: %[[S:.*]] = alloca %struct.SMem, align 8
+ // LLVM-NEXT: %[[SLOT:.*]] = alloca %struct.SMem, align 8
+ // LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[SLOT]], ptr align 8 %[[S]], i64 24, i1 true)
+ // LLVM-NEXT: call void @take_smem(ptr noundef byval(%struct.SMem) align 8 %[[SLOT]])
+
+ cir.func @return_volatile(%arg0: !cir.ptr<!rec_S>) -> !rec_S {
+ %0 = cir.load volatile align(4) %arg0 : !cir.ptr<!rec_S>, !rec_S
+ cir.return %0 : !rec_S
+ }
+
+ // CHECK-LABEL: cir.func @return_volatile(%arg0: !cir.ptr<!rec_S>) -> !u64i
+ // CHECK-NEXT: %[[SLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_S>
+ // CHECK-NEXT: cir.copy %arg0 align(4) to %[[SLOT]] align(8) volatile : !cir.ptr<!rec_S>
+ // CHECK-NEXT: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_S> -> !cir.ptr<!u64i>
+ // CHECK-NEXT: %[[VAL:.*]] = cir.load align(8) %[[VIEW]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NEXT: cir.return %[[VAL]] : !u64i
+
+ // LLVM-LABEL: define i64 @return_volatile(ptr %0)
+ // LLVM-NEXT: %[[SLOT:.*]] = alloca %struct.S, align 8
+ // LLVM-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[SLOT]], ptr align 4 %0, i64 8, i1 true)
+ // LLVM-NEXT: %[[VAL:.*]] = load i64, ptr %[[SLOT]], align 8
+ // LLVM-NEXT: ret i64 %[[VAL]]
+}
diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-record-packed.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-record-packed.cir
index 90eca9f09b3e5e..7c2cb34747e6d8 100644
--- a/clang/test/CIR/Transforms/abi-lowering/x86_64-record-packed.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-record-packed.cir
@@ -41,7 +41,7 @@ module attributes {
// CHECK: %[[SLOT:.*]] = cir.alloca "coerce"{{.*}}: !cir.ptr<!cir.int<u, 40>>
// CHECK: cir.store %arg0, %[[SLOT]] : !cir.int<u, 40>, !cir.ptr<!cir.int<u, 40>>
// CHECK: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!cir.int<u, 40>> -> !cir.ptr<!rec_FiveShort>
- // CHECK: %{{.+}} = cir.load %[[VIEW]] : !cir.ptr<!rec_FiveShort>, !rec_FiveShort
+ // CHECK-NEXT: cir.return
// The caller allocates the slot the other way round, as the record, and
// reads the register out through a view of it.
@@ -51,8 +51,9 @@ module attributes {
}
// CHECK-LABEL: cir.func{{.*}} @call_five_short(%arg0: !cir.int<u, 40>)
+ // CHECK: %[[IN:.*]] = cir.cast bitcast %{{.+}} : !cir.ptr<!cir.int<u, 40>> -> !cir.ptr<!rec_FiveShort>
// CHECK: %[[OUT:.*]] = cir.alloca "coerce"{{.*}}: !cir.ptr<!rec_FiveShort>
- // CHECK: cir.store %{{.+}}, %[[OUT]] : !rec_FiveShort, !cir.ptr<!rec_FiveShort>
+ // CHECK: cir.copy %[[IN]] align(8) to %[[OUT]] align(1) : !cir.ptr<!rec_FiveShort>
// CHECK: %[[OUTVIEW:.*]] = cir.cast bitcast %[[OUT]] : !cir.ptr<!rec_FiveShort> -> !cir.ptr<!cir.int<u, 40>>
// CHECK: %[[REG:.*]] = cir.load %[[OUTVIEW]] : !cir.ptr<!cir.int<u, 40>>, !cir.int<u, 40>
// CHECK: cir.call @take_five_short(%[[REG]]) : (!cir.int<u, 40>) -> ()
@@ -64,13 +65,12 @@ module attributes {
}
// CHECK-LABEL: cir.func{{.*}} @take_nine(%arg0: !u64i, %arg1: !s8i)
- // CHECK: %[[RECSLOT:.*]] = cir.alloca "coerce"{{.*}}: !cir.ptr<!rec_anon_struct>
// CHECK: %[[TUPLE:.*]] = cir.alloca "coerce"{{.*}}: !cir.ptr<!rec_anon_struct>
// CHECK: %[[LO:.*]] = cir.get_member %[[TUPLE]][0]{{.*}}-> !cir.ptr<!u64i>
// CHECK: cir.store %arg0, %[[LO]] : !u64i, !cir.ptr<!u64i>
// CHECK: %[[HI:.*]] = cir.get_member %[[TUPLE]][1]{{.*}}-> !cir.ptr<!s8i>
// CHECK: cir.store %arg1, %[[HI]] : !s8i, !cir.ptr<!s8i>
- // CHECK: %{{.+}} = cir.cast bitcast %[[RECSLOT]] : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_Nine>
+ // CHECK: %{{.+}} = cir.cast bitcast %[[TUPLE]] : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_Nine>
// A packed record the caller must destroy takes the non-byval form rather
// than byval, which keeps the record's declared alignment where byval would
diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-struct-direct-offset.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-struct-direct-offset.cir
index a572c35fbf0460..0005a3c1473c90 100644
--- a/clang/test/CIR/Transforms/abi-lowering/x86_64-struct-direct-offset.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-struct-direct-offset.cir
@@ -37,7 +37,8 @@ module attributes {
// CIR: %[[GEP:.*]] = cir.ptr_stride %[[U8]], %[[OFF]] : (!cir.ptr<!u8i>, !s64i) -> !cir.ptr<!u8i>
// CIR: %[[CAST:.*]] = cir.cast bitcast %[[GEP]] : !cir.ptr<!u8i> -> !cir.ptr<!s64i>
// CIR: cir.store %arg0, %[[CAST]] : !s64i, !cir.ptr<!s64i>
- // CIR: %{{.*}} = cir.load %[[SLOT]] : !cir.ptr<!rec_HiWord>, !rec_HiWord
+ // CIR: %[[H:.*]] = cir.alloca "h" align(8) : !cir.ptr<!rec_HiWord>
+ // CIR: cir.copy %[[SLOT]] align(8) to %[[H]] align(8) : !cir.ptr<!rec_HiWord>
// The return value is read in place from byte 8 of the local aggregate.
cir.func @give_hiword() -> !rec_HiWord {
@@ -138,7 +139,8 @@ module attributes {
// CIR: %[[CGEP:.*]] = cir.ptr_stride %[[CU8]], %[[COFF]] : (!cir.ptr<!u8i>, !s64i) -> !cir.ptr<!u8i>
// CIR: %[[CCAST:.*]] = cir.cast bitcast %[[CGEP]] : !cir.ptr<!u8i> -> !cir.ptr<!s64i>
// CIR: cir.store %[[CRET]], %[[CCAST]] : !s64i, !cir.ptr<!s64i>
- // CIR: %{{.*}} = cir.load %[[CSLOT]] : !cir.ptr<!rec_HiWord>, !rec_HiWord
+ // CIR: %[[R:.*]] = cir.alloca "r" align(8) : !cir.ptr<!rec_HiWord>
+ // CIR: cir.copy %[[CSLOT]] align(8) to %[[R]] align(8) : !cir.ptr<!rec_HiWord>
// Passing an aggregate argument runs the extraction direction at the call
// site: the operand's byte 8 is read out to form the register.
@@ -148,9 +150,9 @@ module attributes {
}
// CIR-LABEL: cir.func{{.*}} @pass_hiword(%arg0: !s64i)
- // CIR: %[[PREC:.*]] = cir.load %{{.*}} : !cir.ptr<!rec_HiWord>, !rec_HiWord
+ // CIR: %[[PARAM:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!rec_HiWord>
// CIR: %[[PSLOT:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!rec_HiWord>
- // CIR: cir.store %[[PREC]], %[[PSLOT]] : !rec_HiWord, !cir.ptr<!rec_HiWord>
+ // CIR: cir.copy %[[PARAM]] align(8) to %[[PSLOT]] align(8) : !cir.ptr<!rec_HiWord>
// CIR: %[[PU8:.*]] = cir.cast bitcast %[[PSLOT]] : !cir.ptr<!rec_HiWord> -> !cir.ptr<!u8i>
// CIR: %[[POFF:.*]] = cir.const #cir.int<8> : !s64i
// CIR: %[[PGEP:.*]] = cir.ptr_stride %[[PU8]], %[[POFF]] : (!cir.ptr<!u8i>, !s64i) -> !cir.ptr<!u8i>
@@ -175,7 +177,8 @@ module attributes {
// CIR: %[[LGEP:.*]] = cir.ptr_stride %[[LU8]], %[[LOFF]] : (!cir.ptr<!u8i>, !s64i) -> !cir.ptr<!u8i>
// CIR: %[[LCAST:.*]] = cir.cast bitcast %[[LGEP]] : !cir.ptr<!u8i> -> !cir.ptr<!s32i>
// CIR: cir.store %arg0, %[[LCAST]] : !s32i, !cir.ptr<!s32i>
- // CIR: %{{.*}} = cir.load %[[LSLOT]] : !cir.ptr<!rec_LeadPad>, !rec_LeadPad
+ // CIR: %[[L:.*]] = cir.alloca "l" align(4) : !cir.ptr<!rec_LeadPad>
+ // CIR: cir.copy %[[LSLOT]] align(4) to %[[L]] align(4) : !cir.ptr<!rec_LeadPad>
}
// LLVM: define i64 @take_hiword(i64 %{{.+}})
diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-struct-direct.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-struct-direct.cir
index 5a8ba5308192ac..2c9695e0ee88a8 100644
--- a/clang/test/CIR/Transforms/abi-lowering/x86_64-struct-direct.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-struct-direct.cir
@@ -26,7 +26,7 @@ module attributes {
// A two-int struct (8 bytes) is coerced to a single i64 register. The body
// reads a field, so the prologue that reconstitutes the record from the
// register is visible: the i64 is stored to a coerce slot, the slot is
- // bitcast to the record pointer, and the record is loaded back.
+ // bitcast to the record pointer, and the body's own slot is copied from it.
cir.func @take_pair(%arg0: !rec_Pair) -> !s32i {
%0 = cir.alloca "p" align(4) : !cir.ptr<!rec_Pair>
cir.store %arg0, %0 : !rec_Pair, !cir.ptr<!rec_Pair>
@@ -39,15 +39,14 @@ module attributes {
// CHECK: %[[SLOT:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!u64i>
// CHECK: cir.store %arg0, %[[SLOT]] : !u64i, !cir.ptr<!u64i>
// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_Pair>
- // CHECK: %[[REC:.*]] = cir.load %[[CAST]] : !cir.ptr<!rec_Pair>, !rec_Pair
// CHECK: %[[LOCAL:.*]] = cir.alloca "p" {{.*}} : !cir.ptr<!rec_Pair>
- // CHECK: cir.store %[[REC]], %[[LOCAL]] : !rec_Pair, !cir.ptr<!rec_Pair>
+ // CHECK: cir.copy %[[CAST]] align(8) to %[[LOCAL]] align(4) : !cir.ptr<!rec_Pair>
// CHECK: %[[FLD:.*]] = cir.get_member %[[LOCAL]][0] {name = "a"} : !cir.ptr<!rec_Pair> -> !cir.ptr<!s32i>
// CHECK: %{{.*}} = cir.load %[[FLD]] : !cir.ptr<!s32i>, !s32i
// A two-i64 struct (16 bytes) is passed as two i64 registers (flattened).
// The body reads the second field: the two registers are written into a
- // flattened coerce slot, reloaded as the record, then indexed.
+ // coerce slot, which the body's own slot is copied from, then indexed.
cir.func @take_two(%arg0: !rec_Two) -> !s64i {
%0 = cir.alloca "t" align(8) : !cir.ptr<!rec_Two>
cir.store %arg0, %0 : !rec_Two, !cir.ptr<!rec_Two>
@@ -57,14 +56,15 @@ module attributes {
}
// CHECK: cir.func{{.*}} @take_two(%arg0: !s64i, %arg1: !s64i) -> !s64i
- // CHECK: cir.alloca "coerce" {{.*}} : !cir.ptr<!rec_anon_struct>
// CHECK: %[[FLAT:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!rec_anon_struct>
// CHECK: %[[E0:.*]] = cir.get_member %[[FLAT]][0] {{.*}} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!s64i>
// CHECK: cir.store %arg0, %[[E0]] : !s64i, !cir.ptr<!s64i>
// CHECK: %[[E1:.*]] = cir.get_member %[[FLAT]][1] {{.*}} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!s64i>
// CHECK: cir.store %arg1, %[[E1]] : !s64i, !cir.ptr<!s64i>
- // CHECK: %{{.*}} = cir.cast bitcast %{{.*}} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_Two>
- // CHECK: %{{.*}} = cir.get_member %{{.*}}[1] {name = "b"} : !cir.ptr<!rec_Two> -> !cir.ptr<!s64i>
+ // CHECK: %[[VIEW:.*]] = cir.cast bitcast %[[FLAT]] : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_Two>
+ // CHECK: %[[T:.*]] = cir.alloca "t" {{.*}} : !cir.ptr<!rec_Two>
+ // CHECK: cir.copy %[[VIEW]] align(8) to %[[T]] align(8) : !cir.ptr<!rec_Two>
+ // CHECK: %{{.*}} = cir.get_member %[[T]][1] {name = "b"} : !cir.ptr<!rec_Two> -> !cir.ptr<!s64i>
// Mixed-alignment fields (i8 then i32) still fit one eightbyte: the i32 is
// placed at its aligned offset and the struct coerces to a single i64.
@@ -76,7 +76,7 @@ module attributes {
// CHECK: %[[SLOT:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!u64i>
// CHECK: cir.store %arg0, %[[SLOT]] : !u64i, !cir.ptr<!u64i>
// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_Mixed>
- // CHECK: %{{.*}} = cir.load %[[CAST]] : !cir.ptr<!rec_Mixed>, !rec_Mixed
+ // CHECK-NEXT: cir.return
// A struct wrapping a small char array is data, not an empty record: the
// 4-byte struct coerces to a single i32.
@@ -88,7 +88,7 @@ module attributes {
// CHECK: %[[SLOT:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!u32i>
// CHECK: cir.store %arg0, %[[SLOT]] : !u32i, !cir.ptr<!u32i>
// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u32i> -> !cir.ptr<!rec_CharBuf>
- // CHECK: %{{.*}} = cir.load %[[CAST]] : !cir.ptr<!rec_CharBuf>, !rec_CharBuf
+ // CHECK-NEXT: cir.return
// A one-byte struct wrapping a single-element char array is data, not an
// empty class: it coerces to a single i8.
@@ -100,17 +100,16 @@ module attributes {
// CHECK: %[[SLOT:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!s8i>
// CHECK: cir.store %arg0, %[[SLOT]] : !s8i, !cir.ptr<!s8i>
// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!s8i> -> !cir.ptr<!rec_Char1>
- // CHECK: %{{.*}} = cir.load %[[CAST]] : !cir.ptr<!rec_Char1>, !rec_Char1
+ // CHECK-NEXT: cir.return
// A 12-byte array argument is passed as two eightbytes (i64 then i32). The
- // two registers are written into a flattened coerce slot, then reloaded as
- // the array.
+ // two registers are written into a coerce slot, then reloaded as the
+ // array.
cir.func @take_arr3(%arg0: !cir.array<!s32i x 3>) {
cir.return
}
// CHECK: cir.func{{.*}} @take_arr3(%arg0: !u64i, %arg1: !s32i)
- // CHECK: cir.alloca "coerce" {{.*}} : !cir.ptr<!rec_anon_struct1>
// CHECK: %[[FLAT:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!rec_anon_struct1>
// CHECK: %[[E0:.*]] = cir.get_member %[[FLAT]][0] {{.*}} : !cir.ptr<!rec_anon_struct1> -> !cir.ptr<!u64i>
// CHECK: cir.store %arg0, %[[E0]] : !u64i, !cir.ptr<!u64i>
@@ -128,11 +127,11 @@ module attributes {
// CHECK: %[[SLOT:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!u64i>
// CHECK: cir.store %arg0, %[[SLOT]] : !u64i, !cir.ptr<!u64i>
// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_anon_struct2>
- // CHECK: %{{.*}} = cir.load %[[CAST]] : !cir.ptr<!rec_anon_struct2>, !rec_anon_struct2
+ // CHECK-NEXT: cir.return
- // The call site round-trips the record through the coercion type: the
- // incoming i64 is reloaded as the record, then stored and reloaded as the
- // i64 the callee expects.
+ // A caller forwarding its own Pair argument stores the incoming i64 to a
+ // coerce slot, copies that slot as a Pair into a second coerce slot, and
+ // loads the i64 it passes to take_pair from the second slot.
cir.func @call_pair(%arg0: !rec_Pair) {
%0 = cir.call @take_pair(%arg0) : (!rec_Pair) -> !s32i
cir.return
@@ -142,9 +141,8 @@ module attributes {
// CHECK: %[[S0:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!u64i>
// CHECK: cir.store %arg0, %[[S0]] : !u64i, !cir.ptr<!u64i>
// CHECK: %[[C0:.*]] = cir.cast bitcast %[[S0]] : !cir.ptr<!u64i> -> !cir.ptr<!rec_Pair>
- // CHECK: %[[REC:.*]] = cir.load %[[C0]] : !cir.ptr<!rec_Pair>, !rec_Pair
// CHECK: %[[S1:.*]] = cir.alloca "coerce" {{.*}} : !cir.ptr<!rec_Pair>
- // CHECK: cir.store %[[REC]], %[[S1]] : !rec_Pair, !cir.ptr<!rec_Pair>
+ // CHECK: cir.copy %[[C0]] align(8) to %[[S1]] align(4) : !cir.ptr<!rec_Pair>
// CHECK: %[[C1:.*]] = cir.cast bitcast %[[S1]] : !cir.ptr<!rec_Pair> -> !cir.ptr<!u64i>
// CHECK: %[[U:.*]] = cir.load %[[C1]] : !cir.ptr<!u64i>, !u64i
// CHECK: %{{.*}} = cir.call @take_pair(%[[U]]) : (!u64i) -> !s32i
diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-struct-padded.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-struct-padded.cir
index de30f61c0a3291..69fed26eb0e9cc 100644
--- a/clang/test/CIR/Transforms/abi-lowering/x86_64-struct-padded.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-struct-padded.cir
@@ -59,8 +59,8 @@ module attributes {
} {
// The coerce slot is allocated as the whole 16-byte record and bitcast down to
- // the 4-byte coerce type, not the other way round, or the reload would read
- // past the slot.
+ // the 4-byte coerce type, not the other way round, or reading the record out
+ // of it would run past the slot.
cir.func @take_ov(%arg0: !rec_Ov) {
cir.return
}
@@ -69,7 +69,7 @@ module attributes {
// CHECK: %[[SLOT:.*]] = cir.alloca "coerce"{{.*}}: !cir.ptr<!rec_Ov>
// CHECK: %[[VIEW:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!rec_Ov> -> !cir.ptr<!s32i>
// CHECK: cir.store %arg0, %[[VIEW]] : !s32i, !cir.ptr<!s32i>
- // CHECK: %{{.+}} = cir.load %[[SLOT]] : !cir.ptr<!rec_Ov>, !rec_Ov
+ // CHECK-NEXT: cir.return
// Past two eightbytes, so byval, and the alignment comes from the record's
// declared alignment in cir.record_layouts rather than from its members,
@@ -97,17 +97,17 @@ module attributes {
// CHECK: cir.func{{.*}} @take_anon_pad(%arg0: !u64i)
- // The caller side coerces in the other direction, storing the record and
- // reloading the register from a view of the same slot.
+ // The caller side coerces in the other direction, copying the whole record
+ // into a slot of its own and reading the register from a view of it.
cir.func @call_ov(%arg0: !rec_Ov) {
cir.call @take_ov(%arg0) : (!rec_Ov) -> ()
cir.return
}
// CHECK: cir.func{{.*}} @call_ov(%arg0: !s32i)
- // CHECK: %[[IN:.*]] = cir.load %{{.+}} : !cir.ptr<!rec_Ov>, !rec_Ov
+ // CHECK: %[[IN:.*]] = cir.alloca "coerce"{{.*}}: !cir.ptr<!rec_Ov>
// CHECK: %[[OUT:.*]] = cir.alloca "coerce"{{.*}}: !cir.ptr<!rec_Ov>
- // CHECK: cir.store %[[IN]], %[[OUT]] : !rec_Ov, !cir.ptr<!rec_Ov>
+ // CHECK: cir.copy %[[IN]] align(4) to %[[OUT]] align(4) : !cir.ptr<!rec_Ov>
// CHECK: %[[OUTVIEW:.*]] = cir.cast bitcast %[[OUT]] : !cir.ptr<!rec_Ov> -> !cir.ptr<!s32i>
// CHECK: %[[REG:.*]] = cir.load %[[OUTVIEW]] : !cir.ptr<!s32i>, !s32i
// CHECK: cir.call @take_ov(%[[REG]]) : (!s32i) -> ()
diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-union.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-union.cir
index a8a14037944d4e..33751c17957b8f 100644
--- a/clang/test/CIR/Transforms/abi-lowering/x86_64-union.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-union.cir
@@ -74,6 +74,19 @@
!rec_UEmptyMarkHoldsData = !cir.union<"UEmptyMarkHoldsData" {empty !rec_NotEmpty, data !s8i}>
!rec_UDataMarkHoldsData = !cir.union<"UDataMarkHoldsData" {data !rec_NotEmpty, data !s8i}>
!rec_SWithNuaEmptyUnion = !cir.struct<"SWithNuaEmptyUnion" {data !rec_UNuaEmptyInt, data !s32i}>
+!rec_UX87Scalar = !cir.union<"UX87Scalar" {data !cir.long_double<!cir.f80>, data !cir.array<!u8i x 11>}>
+!rec_UX87Twice = !cir.union<"UX87Twice" {data !cir.vector<1 x !cir.long_double<!cir.f80>>, data !cir.vector<1 x !cir.long_double<!cir.f80>>}>
+!rec_UX87Bytes = !cir.union<"UX87Bytes" {data !cir.vector<1 x !cir.long_double<!cir.f80>>, data !cir.array<!u8i x 16>}>
+!rec_SX87 = !cir.struct<"SX87" {data !cir.vector<1 x !cir.long_double<!cir.f80>>}>
+!rec_UX87InRec = !cir.union<"UX87InRec" {data !rec_SX87, data !cir.array<!u8i x 16>}>
+!rec_UX87InArr = !cir.union<"UX87InArr" {data !cir.array<!cir.vector<1 x !cir.long_double<!cir.f80>> x 1>, data !cir.array<!u8i x 16>}>
+!rec_SUX87 = !cir.struct<"SUX87" {data !rec_UX87Bytes}>
+!rec_UX87Wide = !cir.union<"UX87Wide" {data !cir.long_double<!cir.f80>, data !s128i}>
+!rec_SLongDouble = !cir.struct<"SLongDouble" {data !cir.long_double<!cir.f80>}>
+!rec_UX87Nested = !cir.union<"UX87Nested" {data !rec_SLongDouble, data !cir.array<!u8i x 4>}>
+!rec_UEmptyX87 = !cir.union<"UEmptyX87" {data !s32i, empty !cir.array<!cir.long_double<!cir.f80> x 0>}, padding = {!cir.array<!u8i x 16>}>
+!rec_UX87Displaced = !cir.union<"UX87Displaced" {data !rec_UEmptyX87, data !cir.long_double<!cir.f80>}>
+!rec_UX87Complex = !cir.union<"UX87Complex" {data !cir.complex<!cir.long_double<!cir.f80>>, data !cir.array<!u8i x 32>}>
module attributes {
cir.triple = "x86_64-unknown-linux-gnu",
@@ -151,7 +164,7 @@ module attributes {
// CHECK: %[[SLOT:.*]] = cir.alloca "coerce" align(4) : !cir.ptr<!s32i>
// CHECK: cir.store %arg0, %[[SLOT]] : !s32i, !cir.ptr<!s32i>
// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!s32i> -> !cir.ptr<!rec_UIntFloat>
- // CHECK: %{{.*}} = cir.load %[[CAST]] : !cir.ptr<!rec_UIntFloat>, !rec_UIntFloat
+ // CHECK-NEXT: cir.return
// Same members as take_int_float in the opposite declaration order. The
// classifier merges eightbyte classes across every member (INTEGER beats
@@ -224,7 +237,6 @@ module attributes {
}
// CHECK: cir.func{{.*}} @take_two_eightbytes(%arg0: !s64i, %arg1: !u64i)
- // CHECK: cir.alloca "coerce" align(8) : !cir.ptr<!rec_anon_struct>
// CHECK: %[[FLAT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_anon_struct>
// CHECK: %[[E0:.*]] = cir.get_member %[[FLAT]][0] {{.*}} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!s64i>
// CHECK: cir.store %arg0, %[[E0]] : !s64i, !cir.ptr<!s64i>
@@ -267,6 +279,174 @@ module attributes {
// CHECK: cir.func{{.*}} @take_int384(%arg0: !cir.ptr<!rec_UInt384> {llvm.align = 16 : i64, llvm.byval = !rec_UInt384, llvm.noundef})
+ // The spill of a parameter passed in registers is a copy out of the
+ // coercion slot.
+ cir.func @take_union_x87_scalar(%arg0: !rec_UX87Scalar) {
+ %0 = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87Scalar>
+ cir.store %arg0, %0 : !rec_UX87Scalar, !cir.ptr<!rec_UX87Scalar>
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @take_union_x87_scalar(%arg0: !u64i, %arg1: !u64i)
+ // CHECK: %[[X87SLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<![[PAIR:rec_anon_struct[0-9]*]]>
+ // CHECK: %[[X87VIEW:.*]] = cir.cast bitcast %[[X87SLOT]] : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87Scalar>
+ // CHECK: %[[X87SPILL:.*]] = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87Scalar>
+ // CHECK-NEXT: cir.copy %[[X87VIEW]] align(16) to %[[X87SPILL]] align(16) : !cir.ptr<!rec_UX87Scalar>
+ // CHECK-NEXT: cir.return
+
+ // Two one-element vectors pass in one SSE register, classified SSE plus
+ // SSEUP.
+ cir.func @take_union_x87_twice(%arg0: !rec_UX87Twice) {
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @take_union_x87_twice(%arg0: !cir.vector<2 x !cir.double>)
+
+ cir.func @take_union_x87_bytes(%arg0: !rec_UX87Bytes) {
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @take_union_x87_bytes(%arg0: !u64i, %arg1: !u64i)
+
+ cir.func @take_union_x87_in_rec(%arg0: !rec_UX87InRec) {
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @take_union_x87_in_rec(%arg0: !u64i, %arg1: !u64i)
+
+ cir.func @take_union_x87_in_arr(%arg0: !rec_UX87InArr) {
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @take_union_x87_in_arr(%arg0: !u64i, %arg1: !u64i)
+
+ cir.func @take_struct_union_x87(%arg0: !rec_SUX87) {
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @take_struct_union_x87(%arg0: !u64i, %arg1: !u64i)
+
+ // The argument is read out of a copy of the operand's storage.
+ cir.func @call_union_x87_bytes(%arg0: !cir.ptr<!rec_UX87Bytes>) {
+ %0 = cir.load %arg0 : !cir.ptr<!rec_UX87Bytes>, !rec_UX87Bytes
+ cir.call @take_union_x87_bytes(%0) : (!rec_UX87Bytes) -> ()
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @call_union_x87_bytes(%arg0: !cir.ptr<!rec_UX87Bytes>)
+ // CHECK: %[[BSLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_UX87Bytes>
+ // CHECK: cir.copy %arg0 align(16) to %[[BSLOT]] align(16) : !cir.ptr<!rec_UX87Bytes>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
+ // CHECK: %[[BVIEW:.*]] = cir.cast bitcast %[[BSLOT]] : !cir.ptr<!rec_UX87Bytes> -> !cir.ptr<![[PAIR]]>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
+ // CHECK: %[[BLOP:.*]] = cir.get_member %[[BVIEW]][0] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
+ // CHECK: %[[BLO:.*]] = cir.load %[[BLOP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
+ // CHECK: %[[BHIP:.*]] = cir.get_member %[[BVIEW]][1] {name = ""} : !cir.ptr<![[PAIR]]> -> !cir.ptr<!u64i>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
+ // CHECK: %[[BHI:.*]] = cir.load %[[BHIP]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
+ // CHECK: cir.call @take_union_x87_bytes(%[[BLO]], %[[BHI]]) : (!u64i, !u64i) -> ()
+
+ // The call result is copied out of the coercion slot it is returned into.
+ cir.func private @return_union_x87_bytes() -> !rec_UX87Bytes
+
+ cir.func @store_union_x87_bytes(%arg0: !cir.ptr<!rec_UX87Bytes>) {
+ %0 = cir.call @return_union_x87_bytes() : () -> !rec_UX87Bytes
+ cir.store %0, %arg0 : !rec_UX87Bytes, !cir.ptr<!rec_UX87Bytes>
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @store_union_x87_bytes(%arg0: !cir.ptr<!rec_UX87Bytes>)
+ // CHECK: %[[RSLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<![[PAIR]]>
+ // CHECK: %[[RES:.*]] = cir.call @return_union_x87_bytes() : () -> ![[PAIR]]
+ // CHECK: cir.store %[[RES]], %[[RSLOT]] : ![[PAIR]], !cir.ptr<![[PAIR]]>
+ // CHECK: %[[RVIEW:.*]] = cir.cast bitcast %[[RSLOT]] : !cir.ptr<![[PAIR]]> -> !cir.ptr<!rec_UX87Bytes>
+ // CHECK: cir.copy %[[RVIEW]] align(16) to %arg0 align(16) : !cir.ptr<!rec_UX87Bytes>
+ // CHECK-NEXT: cir.return
+
+ cir.func @take_union_x87_wide(%arg0: !rec_UX87Wide) {
+ %0 = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87Wide>
+ cir.store %arg0, %0 : !rec_UX87Wide, !cir.ptr<!rec_UX87Wide>
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @take_union_x87_wide(%arg0: !s128i)
+ // CHECK: %[[WSLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<!s128i>
+ // CHECK: cir.store %arg0, %[[WSLOT]] : !s128i, !cir.ptr<!s128i>
+ // CHECK: %[[WVIEW:.*]] = cir.cast bitcast %[[WSLOT]] : !cir.ptr<!s128i> -> !cir.ptr<!rec_UX87Wide>
+ // CHECK: %[[WSPILL:.*]] = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87Wide>
+ // CHECK-NEXT: cir.copy %[[WVIEW]] align(16) to %[[WSPILL]] align(16) : !cir.ptr<!rec_UX87Wide>
+
+ cir.func @call_union_x87_wide(%arg0: !cir.ptr<!rec_UX87Wide>) {
+ %0 = cir.load %arg0 : !cir.ptr<!rec_UX87Wide>, !rec_UX87Wide
+ cir.call @take_union_x87_wide(%0) : (!rec_UX87Wide) -> ()
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @call_union_x87_wide(%arg0: !cir.ptr<!rec_UX87Wide>)
+ // CHECK: %[[WCSLOT:.*]] = cir.alloca "coerce" align(16) : !cir.ptr<!rec_UX87Wide>
+ // CHECK: cir.copy %arg0 align(16) to %[[WCSLOT]] align(16) : !cir.ptr<!rec_UX87Wide>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
+ // CHECK: %[[WCVIEW:.*]] = cir.cast bitcast %[[WCSLOT]] : !cir.ptr<!rec_UX87Wide> -> !cir.ptr<!s128i>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
+ // CHECK: %[[WCARG:.*]] = cir.load align(16) %[[WCVIEW]] : !cir.ptr<!s128i>, !s128i
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
+ // CHECK: cir.call @take_union_x87_wide(%[[WCARG]]) : (!s128i) -> ()
+
+ // Here the second eightbyte is X87UP with no X87 before it, so the union goes
+ // to memory.
+ cir.func @take_union_x87_nested(%arg0: !rec_UX87Nested) {
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @take_union_x87_nested(%arg0: !cir.ptr<!rec_UX87Nested> {llvm.align = 16 : i64, llvm.byval = !rec_UX87Nested, llvm.noundef})
+
+ // The integer in the first member makes the first eightbyte INTEGER, so the
+ // second is X87UP with no X87 before it and the union goes to memory.
+ cir.func @take_union_x87_displaced(%arg0: !rec_UX87Displaced) {
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @take_union_x87_displaced(%arg0: !cir.ptr<!rec_UX87Displaced> {llvm.align = 16 : i64, llvm.byval = !rec_UX87Displaced, llvm.noundef})
+
+ // At 32 bytes the union goes to memory, both as a parameter and as a return
+ // value. A parameter stored twice has no single spill to fill, so each
+ // store becomes a copy from the incoming pointer.
+ cir.func @take_union_x87_complex(%arg0: !rec_UX87Complex, %arg1: !cir.ptr<!rec_UX87Complex>) {
+ %0 = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87Complex>
+ cir.store %arg0, %0 : !rec_UX87Complex, !cir.ptr<!rec_UX87Complex>
+ cir.store %arg0, %arg1 : !rec_UX87Complex, !cir.ptr<!rec_UX87Complex>
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @take_union_x87_complex(%arg0: !cir.ptr<!rec_UX87Complex> {llvm.align = 16 : i64, llvm.byval = !rec_UX87Complex, llvm.noundef}, %arg1: !cir.ptr<!rec_UX87Complex>)
+ // CHECK: %[[CSPILL:.*]] = cir.alloca "u" align(16) : !cir.ptr<!rec_UX87Complex>
+ // CHECK-NEXT: cir.copy %arg0 align(16) to %[[CSPILL]] align(16) : !cir.ptr<!rec_UX87Complex>
+ // CHECK-NEXT: cir.copy %arg0 align(16) to %arg1 align(16) : !cir.ptr<!rec_UX87Complex>
+ // CHECK-NEXT: cir.return
+
+ cir.func private @return_union_x87_complex() -> !rec_UX87Complex
+
+ // CHECK: cir.func private @return_union_x87_complex(!cir.ptr<!rec_UX87Complex> {llvm.align = 16 : i64, llvm.dead_on_unwind, llvm.sret = !rec_UX87Complex, llvm.writable})
+
+ // The result is stored where the call cannot be handed the destination, so
+ // it is copied out of a return slot of its own.
+ cir.func @store_union_x87_complex(%arg0: !cir.ptr<!cir.ptr<!rec_UX87Complex>>) {
+ %0 = cir.call @return_union_x87_complex() : () -> !rec_UX87Complex
+ %1 = cir.load %arg0 : !cir.ptr<!cir.ptr<!rec_UX87Complex>>, !cir.ptr<!rec_UX87Complex>
+ cir.store %0, %1 : !rec_UX87Complex, !cir.ptr<!rec_UX87Complex>
+ cir.return
+ }
+
+ // CHECK: cir.func{{.*}} @store_union_x87_complex(%arg0: !cir.ptr<!cir.ptr<!rec_UX87Complex>>)
+ // CHECK: %[[SRET:.*]] = cir.alloca "sret" align(16) : !cir.ptr<!rec_UX87Complex>
+ // CHECK: cir.call @return_union_x87_complex(%[[SRET]])
+ // CHECK: %[[DEST:.*]] = cir.load %arg0 : !cir.ptr<!cir.ptr<!rec_UX87Complex>>, !cir.ptr<!rec_UX87Complex>
+ // CHECK-NEXT: cir.copy %[[SRET]] align(16) to %[[DEST]] align(16) : !cir.ptr<!rec_UX87Complex>
+ // CHECK-NEXT: cir.return
+
// The byval alignment comes from the record's declared alignment, which the
// layout metadata carries because the members alone cannot express an
// alignment attribute. Same members as take_big, alignment 32 rather than 8.
@@ -447,7 +627,7 @@ module attributes {
// CHECK: %[[SLOT:.*]] = cir.alloca "coerce" align(4) : !cir.ptr<!u32i>
// CHECK: cir.store %arg0, %[[SLOT]] : !u32i, !cir.ptr<!u32i>
// CHECK: %[[CAST:.*]] = cir.cast bitcast %[[SLOT]] : !cir.ptr<!u32i> -> !cir.ptr<!rec_UBitArrayUnit>
- // CHECK: %{{.*}} = cir.load %[[CAST]] : !cir.ptr<!rec_UBitArrayUnit>, !rec_UBitArrayUnit
+ // CHECK-NEXT: cir.return
// Each variant of a union is its own access unit, so the two bit-fields here
// do not share one. The widest declaration among them is what counts.
@@ -497,8 +677,9 @@ module attributes {
}
// CHECK: cir.func{{.*}} @ret_bit_array_unit(%arg0: !u32i) -> !u32i
- // CHECK: %[[RETSLOT:.*]] = cir.alloca "u" align(4) : !cir.ptr<!rec_UBitArrayUnit>
- // CHECK-NEXT: cir.store %{{.*}}, %[[RETSLOT]] : !rec_UBitArrayUnit, !cir.ptr<!rec_UBitArrayUnit>
+ // CHECK: %[[ARGVIEW:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!u32i> -> !cir.ptr<!rec_UBitArrayUnit>
+ // CHECK-NEXT: %[[RETSLOT:.*]] = cir.alloca "u" align(4) : !cir.ptr<!rec_UBitArrayUnit>
+ // CHECK-NEXT: cir.copy %[[ARGVIEW]] align(4) to %[[RETSLOT]] align(1) : !cir.ptr<!rec_UBitArrayUnit>
// CHECK-NEXT: %[[RETCAST:.*]] = cir.cast bitcast %[[RETSLOT]] : !cir.ptr<!rec_UBitArrayUnit> -> !cir.ptr<!u32i>
// CHECK-NEXT: %[[RETVAL:.*]] = cir.load align(4) %[[RETCAST]] : !cir.ptr<!u32i>, !u32i
// CHECK-NEXT: cir.return %[[RETVAL]] : !u32i
@@ -509,7 +690,10 @@ module attributes {
}
// CHECK: cir.func{{.*}} @call_bit_array_unit(%arg0: !u32i)
- // CHECK: %[[ARGVAL:.*]] = cir.load %{{.*}} : !cir.ptr<!u32i>, !u32i
+ // CHECK: cir.copy %{{[^ ]+}} align(4) to %{{[^ ]+}} align(1) : !cir.ptr<!rec_UBitArrayUnit>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
+ // CHECK: %[[ARGVAL:.*]] = cir.load %{{[^ ]+}} : !cir.ptr<!u32i>, !u32i
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
// CHECK: cir.call @take_bit_array_unit(%[[ARGVAL]]) : (!u32i) -> ()
// Both eightbytes classify INTEGER and are flattened into one argument each,
@@ -673,7 +857,9 @@ module attributes {
}
// CHECK: cir.func{{.*}} @call_nua_empty_int(%arg0: !s32i)
- // CHECK: cir.call @take_nua_empty_int(%{{.*}}) : (!s32i) -> ()
+ // CHECK: cir.copy %{{[^ ]+}} align(4) to %{{[^ ]+}} align(4) : !cir.ptr<!rec_UNuaEmptyInt>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
+ // CHECK: cir.call @take_nua_empty_int(%{{[^,)]+}}) : (!s32i) -> ()
cir.func @call_nua_big_empty_int(%arg0: !rec_UNuaBigEmptyInt) {
cir.call @take_nua_big_empty_int(%arg0) : (!rec_UNuaBigEmptyInt) -> ()
@@ -683,6 +869,7 @@ module attributes {
// CHECK: cir.func{{.*}} @call_nua_big_empty_int(%arg0: !cir.ptr<!rec_UNuaBigEmptyInt> {llvm.align = 32 : i64, llvm.byval = !rec_UNuaBigEmptyInt, llvm.noundef})
// CHECK: %[[BYVAL:.*]] = cir.alloca "byval" align(32) : !cir.ptr<!rec_UNuaBigEmptyInt>
// CHECK: cir.copy %arg0 align(32) to %[[BYVAL]] align(32) : !cir.ptr<!rec_UNuaBigEmptyInt>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
// CHECK: cir.call @take_nua_big_empty_int(%[[BYVAL]]) : (!cir.ptr<!rec_UNuaBigEmptyInt> {llvm.align = 32 : i64, llvm.byval = !rec_UNuaBigEmptyInt, llvm.noundef}) -> ()
cir.func @ret_nua_empty_floats(%arg0: !rec_UNuaEmptyFloats) -> !rec_UNuaEmptyFloats {
@@ -711,7 +898,9 @@ module attributes {
}
// CHECK: cir.func{{.*}} @call_nua_empty_two_eightbytes(%arg0: !u64i, %arg1: !u64i)
- // CHECK: cir.call @take_nua_empty_two_eightbytes(%{{.*}}, %{{.*}}) : (!u64i, !u64i) -> ()
+ // CHECK: cir.copy %{{[^ ]+}} align(8) to %{{[^ ]+}} align(1) : !cir.ptr<!rec_UNuaEmptyTwoEightbytes>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
+ // CHECK: cir.call @take_nua_empty_two_eightbytes(%{{[^,)]+}}, %{{[^,)]+}}) : (!u64i, !u64i) -> ()
// A coerced union return round-trips through the coercion type.
cir.func @ret_empty_int(%arg0: !rec_UEmptyInt) -> !rec_UEmptyInt {
@@ -741,7 +930,9 @@ module attributes {
}
// CHECK: cir.func{{.*}} @call_empty_int(%arg0: !s32i)
- // CHECK: cir.call @take_empty_int(%{{.*}}) : (!s32i) -> ()
+ // CHECK: cir.copy %{{[^ ]+}} align(4) to %{{[^ ]+}} align(4) : !cir.ptr<!rec_UEmptyInt>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
+ // CHECK: cir.call @take_empty_int(%{{[^,)]+}}) : (!s32i) -> ()
// A union the record layout marks as unable to pass in registers goes
// indirect without byval, however small it is.
@@ -778,8 +969,9 @@ module attributes {
}
// CHECK: cir.func{{.*}} @ret_int_float(%arg0: !s32i) -> !s32i
- // CHECK: %[[RETSLOT:.*]] = cir.alloca "u" align(4) : !cir.ptr<!rec_UIntFloat>
- // CHECK-NEXT: cir.store %{{.*}}, %[[RETSLOT]] : !rec_UIntFloat, !cir.ptr<!rec_UIntFloat>
+ // CHECK: %[[ARGVIEW:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!s32i> -> !cir.ptr<!rec_UIntFloat>
+ // CHECK-NEXT: %[[RETSLOT:.*]] = cir.alloca "u" align(4) : !cir.ptr<!rec_UIntFloat>
+ // CHECK-NEXT: cir.copy %[[ARGVIEW]] align(4) to %[[RETSLOT]] align(4) : !cir.ptr<!rec_UIntFloat>
// CHECK-NEXT: %[[RETCAST:.*]] = cir.cast bitcast %[[RETSLOT]] : !cir.ptr<!rec_UIntFloat> -> !cir.ptr<!s32i>
// CHECK-NEXT: %[[RET:.*]] = cir.load align(4) %[[RETCAST]] : !cir.ptr<!s32i>, !s32i
// CHECK-NEXT: cir.return %[[RET]] : !s32i
@@ -805,10 +997,14 @@ module attributes {
}
// CHECK: cir.func{{.*}} @call_int_float(%arg0: !s32i)
+ // CHECK: %[[PARAMVIEW:.*]] = cir.cast bitcast %{{.*}} : !cir.ptr<!s32i> -> !cir.ptr<!rec_UIntFloat>
// CHECK: %[[ARGSLOT:.*]] = cir.alloca "coerce" align(4) : !cir.ptr<!rec_UIntFloat>
- // CHECK: cir.store %{{.*}}, %[[ARGSLOT]] : !rec_UIntFloat, !cir.ptr<!rec_UIntFloat>
+ // CHECK: cir.copy %[[PARAMVIEW]] align(4) to %[[ARGSLOT]] align(4) : !cir.ptr<!rec_UIntFloat>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
// CHECK: %[[ARGCAST:.*]] = cir.cast bitcast %[[ARGSLOT]] : !cir.ptr<!rec_UIntFloat> -> !cir.ptr<!s32i>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
// CHECK: %[[ARG:.*]] = cir.load %[[ARGCAST]] : !cir.ptr<!s32i>, !s32i
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
// CHECK: cir.call @take_int_float(%[[ARG]]) : (!s32i) -> ()
// Both eightbytes are decomposed at the call site.
@@ -819,102 +1015,251 @@ module attributes {
// CHECK: cir.func{{.*}} @call_two_eightbytes(%arg0: !s64i, %arg1: !u64i)
// CHECK: %[[CALLSLOT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_UTwoEightbytes>
+ // CHECK: cir.copy %{{[^ ]+}} align(8) to %[[CALLSLOT]] align(8) : !cir.ptr<!rec_UTwoEightbytes>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
// CHECK: %[[CALLCAST:.*]] = cir.cast bitcast %[[CALLSLOT]] : !cir.ptr<!rec_UTwoEightbytes> -> !cir.ptr<!rec_anon_struct>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
// CHECK: %[[G0:.*]] = cir.get_member %[[CALLCAST]][0] {{.*}} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!s64i>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
// CHECK: %[[A0:.*]] = cir.load %[[G0]] : !cir.ptr<!s64i>, !s64i
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
// CHECK: %[[G1:.*]] = cir.get_member %[[CALLCAST]][1] {{.*}} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!u64i>
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
// CHECK: %[[A1:.*]] = cir.load %[[G1]] : !cir.ptr<!u64i>, !u64i
+ // CHECK-NOT: {{cir.load .*, !rec_|cir.store .* : !rec_}}
// CHECK: cir.call @take_two_eightbytes(%[[A0]], %[[A1]]) : (!s64i, !u64i) -> ()
}
-// LLVM: define void @take_int_float(i32 %{{.+}})
+// LLVM: define void @take_int_float(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// Declaration order does not reach the lowered signature: the CIR-level
// signedness difference from the reduction tie disappears here, and both
// orders land on the same integer register the class merge picked.
-// LLVM: define void @take_float_int(i32 %{{.+}})
-// LLVM: define void @take_long_double(i64 %{{.+}})
-// LLVM: define void @take_double_long(i64 %{{.+}})
-// LLVM: define void @take_floats(float %{{.+}})
-// LLVM: define void @take_three(i24 %{{.+}})
-// LLVM: define void @take_narrow_storage(i64 %{{.+}})
-// LLVM: define void @take_two_eightbytes(i64 %{{.+}}, i64 %{{.+}})
-// LLVM: define void @take_big(ptr noundef byval(%union.UBig) align 8 %{{.+}})
-// LLVM: define void @take_tail_pad(ptr noundef byval(%union.UTailPad) align 8 %{{.+}})
+// LLVM: define void @take_float_int(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_long_double(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_double_long(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_floats(float %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_three(i24 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_narrow_storage(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_two_eightbytes(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_big(ptr noundef byval(%union.UBig) align 8 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_tail_pad(ptr noundef byval(%union.UTailPad) align 8 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: declare void @return_tail_pad(ptr dead_on_unwind writable sret(%union.UTailPad) align 8)
-// LLVM: define void @take_tail_pad_no_regs(ptr nofreeobj noundef align 8 dereferenceable(24) %{{.+}})
-// LLVM: define void @take_int384(ptr noundef byval(%union.UInt384) align 16 %{{.+}})
-// LLVM: define void @take_big_over_aligned(ptr noundef byval(%union.UBigOverAligned) align 32 %{{.+}})
-// LLVM: define void @take_struct_over_aligned(ptr noundef byval(%struct.SOverAligned) align 32 %{{.+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_tail_pad_no_regs(ptr nofreeobj noundef align 8 dereferenceable(24) %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_int384(ptr noundef byval(%union.UInt384) align 16 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_union_x87_scalar(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %{{[^,)]+}}, i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_union_x87_twice(<2 x double> %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_union_x87_bytes(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_union_x87_in_rec(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_union_x87_in_arr(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_struct_union_x87(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @call_union_x87_bytes(ptr %[[BSRC:[^,)]+]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %[[BSRC]], i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @take_union_x87_bytes(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: declare { i64, i64 } @return_union_x87_bytes()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @store_union_x87_bytes(ptr %[[BDST:[^,)]+]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call { i64, i64 } @return_union_x87_bytes()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[BDST]], ptr align 16 %{{[^,)]+}}, i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_union_x87_wide(i128 %[[WARG:[^,)]+]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: store i128 %[[WARG]], ptr %[[WSLOT:[^,]+]],
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %[[WSLOT]], i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @call_union_x87_wide(ptr %[[WSRC:[^,)]+]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[WCSLOT:[^,)]+]], ptr align 16 %[[WSRC]], i64 16, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: %[[WCARG:[^ ]+]] = load i128, ptr %[[WCSLOT]], align 16
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @take_union_x87_wide(i128 %[[WCARG]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_union_x87_nested(ptr noundef byval(%union.UX87Nested) align 16 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_union_x87_displaced(ptr noundef byval(%union.UX87Displaced) align 16 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_union_x87_complex(ptr noundef byval(%union.UX87Complex) align 16 %[[CSRC:[^,)]+]], ptr %[[CDST:[^,)]+]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %[[CSRC]], i64 32, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %[[CDST]], ptr align 16 %[[CSRC]], i64 32, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: declare void @return_union_x87_complex(ptr dead_on_unwind writable sret(%union.UX87Complex) align 16)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @store_union_x87_complex(ptr %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @return_union_x87_complex(ptr dead_on_unwind writable sret(%union.UX87Complex) align 16 %[[SRETSLOT:[^,)]+]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 16 %{{[^,)]+}}, ptr align 16 %[[SRETSLOT]], i64 32, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_big_over_aligned(ptr noundef byval(%union.UBigOverAligned) align 32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_struct_over_aligned(ptr noundef byval(%struct.SOverAligned) align 32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_empty()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @ret_empty()
-// LLVM: define void @take_empty_int(i32 %{{.+}})
-// LLVM: define void @take_empty_over(i32 %{{.+}})
+// LLVM: define void @take_empty_int(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_empty_over(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_empty_only()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_empty_over_only()
-// LLVM: define void @take_empty_double(double %{{.+}})
-// LLVM: define void @take_empty_bytes(i64 %{{.+}})
-// LLVM: define void @take_arr_empty(i8 %{{.+}})
-// LLVM: define void @take_arr8_empty(i8 %{{.+}})
-// LLVM: define void @take_big_empty_int(ptr noundef byval(%union.UBigEmptyInt) align 32 %{{.+}})
-// LLVM: define void @take_two_empty(i32 %{{.+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_empty_double(double %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_empty_bytes(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_arr_empty(i8 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_arr8_empty(i8 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_big_empty_int(ptr noundef byval(%union.UBigEmptyInt) align 32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_two_empty(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_empty_unnamed_bits(i32 %{{[^,)]+}})
-// LLVM: define void @take_empty_floats(<2 x float> %{{.+}})
-// LLVM: define void @take_bit_spans(i32 %{{.+}})
-// LLVM: define void @take_bit_plus_long(i64 %{{.+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_empty_floats(<2 x float> %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_bit_spans(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_bit_plus_long(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_bit_extent(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_bit_extent_wide(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_bit_array_unit(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_two_bit_units(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_unnamed_bit_extent(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_bit_empty_span(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_bit_extent_packed(i8 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_overshoot(i16 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define i32 @ret_bit_array_unit(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @call_bit_array_unit(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: call void @take_bit_array_unit(i32 %{{[^,)]+}})
-// LLVM: define void @take_empty_two_eightbytes(i64 %{{.+}}, i64 %{{.+}})
-// LLVM: define void @take_struct_with_empty_union(i64 %{{.+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_empty_two_eightbytes(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_struct_with_empty_union(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_nua_empty_int(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_nua_two_empty(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_nua_empty_only()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_nua_empty_double(double %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_nua_empty_floats(<2 x float> %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_nua_empty_two_eightbytes(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_nua_big_empty_int(ptr noundef byval(%union.UNuaBigEmptyInt) align 32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_nua_empty_over(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_nua_empty_over_only()
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_nua_big_empty_only(ptr noundef byval(%union.UNuaBigEmptyOnly) align 32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_zero_len_arr_union(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_empty_mark_holds_data(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_data_mark_holds_data(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_nua_empty_bit_int(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_nua_empty_no_regs(ptr nofreeobj noundef align 4 dereferenceable(4) %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_struct_with_nua_empty_union(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define i32 @ret_nua_empty_int(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @ret_nua_empty_only()
// LLVM: define void @call_nua_empty_int(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: call void @take_nua_empty_int(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @call_nua_big_empty_int(ptr noundef byval(%union.UNuaBigEmptyInt) align 32 %[[NUABIG:[^,)]+]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 32 %[[NUABIGSLOT:[^,]+]], ptr align 32 %[[NUABIG]], i64 32, i1 false)
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: call void @take_nua_big_empty_int(ptr noundef byval(%union.UNuaBigEmptyInt) align 32 %[[NUABIGSLOT]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define <2 x float> @ret_nua_empty_floats(<2 x float> %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define { i64, i64 } @ret_nua_empty_two_eightbytes(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @call_nua_empty_two_eightbytes(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: call void @take_nua_empty_two_eightbytes(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
-// LLVM: define i32 @ret_empty_int(i32 %{{.+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define i32 @ret_empty_int(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @ret_empty_only()
-// LLVM: define void @call_empty_int(i32 %{{.+}})
-// LLVM: call void @take_empty_int(i32 %{{.+}})
-// LLVM: define void @take_no_regs(ptr nofreeobj noundef align 4 dereferenceable(4) %{{.+}})
+// LLVM: define void @call_empty_int(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @take_empty_int(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_no_regs(ptr nofreeobj noundef align 4 dereferenceable(4) %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: define void @take_bit_no_regs(ptr nofreeobj noundef align 4 dereferenceable(4) %{{[^,)]+}})
-// LLVM: define void @take_struct_with_union(i64 %{{.+}})
-// LLVM: define i32 @ret_int_float(i32 %{{.+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @take_struct_with_union(i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define i32 @ret_int_float(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// The sret pointer is a block argument with no slot behind it, so the copy
// filling it states no alignment of its own and lowering falls back to the
// union's ABI alignment of 1.
-// LLVM: define void @ret_big(ptr dead_on_unwind noalias writable sret(%union.UBig) align 1 %[[UBIGRET:[^,]+]], ptr noundef byval(%union.UBig) align 8 %[[UBIGARG:.+]])
+// LLVM: define void @ret_big(ptr dead_on_unwind noalias writable sret(%union.UBig) align 1 %[[UBIGRET:[^,)]+]], ptr noundef byval(%union.UBig) align 8 %[[UBIGARG:[^,)]+]])
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
// LLVM: call void @llvm.memcpy.p0.p0.i64(ptr align 1 %[[UBIGRET]], ptr align 8 %[[UBIGARG]], i64 32, i1 false)
-// LLVM: define void @call_int_float(i32 %{{.+}})
-// LLVM: call void @take_int_float(i32 %{{.+}})
-// LLVM: define void @call_two_eightbytes(i64 %{{.+}}, i64 %{{.+}})
-// LLVM: call void @take_two_eightbytes(i64 %{{.+}}, i64 %{{.+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @call_int_float(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @take_int_float(i32 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: define void @call_two_eightbytes(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
+// LLVM: call void @take_two_eightbytes(i64 %{{[^,)]+}}, i64 %{{[^,)]+}})
+// LLVM-NOT: {{(load|store)( volatile| atomic)* %(union|struct)\.}}
diff --git a/clang/test/CIR/Transforms/abi-lowering/x86_64-vptr.cir b/clang/test/CIR/Transforms/abi-lowering/x86_64-vptr.cir
index af44f59800ddd0..8511dd31de8654 100644
--- a/clang/test/CIR/Transforms/abi-lowering/x86_64-vptr.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/x86_64-vptr.cir
@@ -89,13 +89,14 @@ module attributes {
}
// CHECK: cir.func{{.*}} @take_vp_long(%arg0: !cir.ptr<!void>, %arg1: !s64i) -> !s64i
- // CHECK: cir.alloca "coerce" align(8) : !cir.ptr<!rec_anon_struct>
// CHECK: %[[FLAT:.*]] = cir.alloca "coerce" align(8) : !cir.ptr<!rec_anon_struct>
// CHECK: %[[E0:.*]] = cir.get_member %[[FLAT]][0] {{.*}} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!cir.ptr<!void>>
// CHECK: cir.store %arg0, %[[E0]] : !cir.ptr<!void>, !cir.ptr<!cir.ptr<!void>>
// CHECK: %[[E1:.*]] = cir.get_member %[[FLAT]][1] {{.*}} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!s64i>
// CHECK: cir.store %arg1, %[[E1]] : !s64i, !cir.ptr<!s64i>
- // CHECK: %{{.*}} = cir.cast bitcast %{{.*}} : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_VPLong>
+ // CHECK: %[[VIEW:.*]] = cir.cast bitcast %[[FLAT]] : !cir.ptr<!rec_anon_struct> -> !cir.ptr<!rec_VPLong>
+ // CHECK: %[[V:.*]] = cir.alloca "v" align(8) : !cir.ptr<!rec_VPLong>
+ // CHECK: cir.copy %[[VIEW]] align(8) to %[[V]] align(8) : !cir.ptr<!rec_VPLong>
// LLVM: define i64 @take_vp_long(ptr %{{[^,]+}}, i64 %{{[^,)]+}})
// Two ints share the second eightbyte behind the vtable pointer, so that
>From 9f9204f277f58dcab24ea23a08431fbca3217c60 Mon Sep 17 00:00:00 2001
From: Adam Smith <adams at nvidia.com>
Date: Wed, 14 Oct 2026 13:45:07 -0700
Subject: [PATCH 2/2] [CIR] Address reviewer feedback
Assisted-by: Cursor / Claude Opus 5.5
---
.../TargetLowering/CIRABIRewriteContext.cpp | 61 ++++++++-----------
.../abi-lowering/coerce-alloca-addrspace.cir | 3 +
2 files changed, 30 insertions(+), 34 deletions(-)
diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
index 7e167b66a32332..23a9f02087fa58 100644
--- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp
@@ -671,21 +671,6 @@ static mlir::Value emitCoercedRecordLoad(
offset);
}
-/// Whether \p ty is a struct that ends in pad members, as CIRGen emits for tail
-/// padding another object may reuse. It is also true of other structs CIRGen
-/// ends in pad members, such as an over-aligned one.
-static bool hasTailPadding(mlir::Type ty) {
- auto structTy = mlir::dyn_cast<cir::StructType>(ty);
- if (!structTy)
- return false;
- llvm::ArrayRef<mlir::Type> members = structTy.getMembers();
- llvm::ArrayRef<cir::RecordMemberKind> kinds = structTy.getMemberKinds();
- for (auto [member, kind] : llvm::reverse(llvm::zip_equal(members, kinds)))
- if (cir::memberOwnsBytes(member))
- return kind == cir::RecordMemberKind::Pad;
- return false;
-}
-
/// Whether a cir.copy of \p ty into \p dst that stands in for a store of it
/// skips tail padding. Unless \p dst is an alloca, which holds a complete
/// object, it can be a potentially-overlapping subobject, such as a
@@ -693,9 +678,19 @@ static bool hasTailPadding(mlir::Type ty) {
/// object. Classic CodeGen writes only the data size into a destination it is
/// handed (EmitCall) and into one that may overlap (EmitAggregateCopy).
/// Copying the whole object into an alloca is equivalent, since nothing else
-/// lives in its tail padding.
+/// lives in its tail padding. A struct has tail padding when its last members
+/// are pad members, which CIRGen emits for padding another object may reuse and
+/// for some other structs, such as an over-aligned one.
static bool shouldSkipTailPadding(mlir::Value dst, mlir::Type ty) {
- return !cir::getUnderlyingAlloca(dst) && hasTailPadding(ty);
+ auto structTy = mlir::dyn_cast<cir::StructType>(ty);
+ if (!structTy || cir::getUnderlyingAlloca(dst))
+ return false;
+ llvm::ArrayRef<mlir::Type> members = structTy.getMembers();
+ llvm::ArrayRef<cir::RecordMemberKind> kinds = structTy.getMemberKinds();
+ for (auto [member, kind] : llvm::reverse(llvm::zip_equal(members, kinds)))
+ if (cir::memberOwnsBytes(member))
+ return kind == cir::RecordMemberKind::Pad;
+ return false;
}
/// Rewire the uses of the record value \p val, whose bytes sit at \p addr,
@@ -729,25 +724,23 @@ rewireRecordUses(mlir::OpBuilder &builder, mlir::Location loc, mlir::Value val,
mlir::Operation *user = use.getOwner();
if (exceptions.contains(user))
continue;
- auto store = mlir::dyn_cast<cir::StoreOp>(user);
- if (!store) {
- auto member = mlir::dyn_cast<cir::ExtractMemberOp>(user);
- bool isFieldRead =
- member && !mlir::isa<cir::RecordType>(member.getType());
- if (!isFieldRead &&
- !mlir::isa<cir::CIRCallOpInterface, cir::ReturnOp>(user))
- return user->emitOpError()
- << "use of a record argument or result other than by a "
- "store, call, return or member read not yet implemented "
- "in CallConvLowering";
- otherUses.push_back(&use);
+ if (auto store = mlir::dyn_cast<cir::StoreOp>(user)) {
+ if (store.getMemOrder())
+ return store.emitOpError()
+ << "atomic store of a record argument or result not yet "
+ "implemented in CallConvLowering";
+ stores.push_back(store);
continue;
}
- if (store.getMemOrder())
- return store.emitOpError()
- << "atomic store of a record argument or result not yet "
- "implemented in CallConvLowering";
- stores.push_back(store);
+ auto member = mlir::dyn_cast<cir::ExtractMemberOp>(user);
+ bool isFieldRead = member && !mlir::isa<cir::RecordType>(member.getType());
+ if (!isFieldRead &&
+ !mlir::isa<cir::CIRCallOpInterface, cir::ReturnOp>(user))
+ return user->emitOpError()
+ << "use of a record argument or result other than by a store, "
+ "call, return or member read not yet implemented in "
+ "CallConvLowering";
+ otherUses.push_back(&use);
}
mlir::IntegerAttr srcAlign = builder.getI64IntegerAttr(addrAlign);
diff --git a/clang/test/CIR/Transforms/abi-lowering/coerce-alloca-addrspace.cir b/clang/test/CIR/Transforms/abi-lowering/coerce-alloca-addrspace.cir
index 2556ab59989755..3f84f83179a9e7 100644
--- a/clang/test/CIR/Transforms/abi-lowering/coerce-alloca-addrspace.cir
+++ b/clang/test/CIR/Transforms/abi-lowering/coerce-alloca-addrspace.cir
@@ -93,6 +93,9 @@ cir.func @takes_dim3(%arg0: !rec_Dim3)
// CHECK: cir.store %[[D0]], %[[P0]] : !s64i, !cir.ptr<!s64i, target_address_space(5)>
// CHECK: %[[P1:.*]] = cir.get_member %[[FSLOT]][1] {{.*}} -> !cir.ptr<!s32i, target_address_space(5)>
// CHECK: cir.store %[[D1]], %[[P1]] : !s32i, !cir.ptr<!s32i, target_address_space(5)>
+// CHECK: cir.cast bitcast %[[FSLOT]] : !cir.ptr<{{.*}}, target_address_space(5)> -> !cir.ptr<!rec_Dim3, target_address_space(5)>
+// CHECK-NOT: cir.load {{.*}}!rec_Dim3
+// CHECK: cir.return
cir.func @call_dim3() attributes { test_classify = #passthrough } {
%0 = cir.const #cir.zero : !rec_Dim3
More information about the cfe-commits
mailing list