[llvm] r307796 - Add element atomic memmove intrinsic
Daniel Neilson via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 12 12:11:48 PDT 2017
The fix is (tested on global.ll test):
diff --git a/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp b/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
index c02ef4a1c39..b1a0956bac5 100644
--- a/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
@@ -400,6 +400,12 @@ RuntimeLibcallSignatures[RTLIB::UNKNOWN_LIBCALL] = {
/* MEMCPY_ELEMENT_ATOMIC_8 */ iPTR_func_iPTR_iPTR_iPTR,
/* MEMCPY_ELEMENT_ATOMIC_16 */ iPTR_func_iPTR_iPTR_iPTR,
+/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1 */ unsupported,
+/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2 */ unsupported,
+/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4 */ unsupported,
+/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8 */ unsupported,
+/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16 */ unsupported,
+
// EXCEPTION HANDLING
/* UNWIND_RESUME */ unsupported,
@@ -844,6 +850,11 @@ RuntimeLibcallNames[RTLIB::UNKNOWN_LIBCALL] = {
/* MEMCPY_ELEMENT_ATOMIC_4 */ "MEMCPY_ELEMENT_ATOMIC_4",
/* MEMCPY_ELEMENT_ATOMIC_8 */ "MEMCPY_ELEMENT_ATOMIC_8",
/* MEMCPY_ELEMENT_ATOMIC_16 */ "MEMCPY_ELEMENT_ATOMIC_16",
+/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1 */ nullptr,
+/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2 */ nullptr,
+/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4 */ nullptr,
+/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8 */ nullptr,
+/* MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16 */ nullptr,
/* UNWIND_RESUME */ "_Unwind_Resume",
/* SYNC_VAL_COMPARE_AND_SWAP_1 */ "__sync_val_compare_and_swap_1",
/* SYNC_VAL_COMPARE_AND_SWAP_2 */ "__sync_val_compare_and_swap_2”,
-Daniel
On Jul 12, 2017, at 12:04 PM, Daniel Neilson <dneilson at azul.com<mailto:dneilson at azul.com>> wrote:
The problem is in lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp. There’s an array declared:
545 static const char *
546 RuntimeLibcallNames[RTLIB::UNKNOWN_LIBCALL] = {
That is defining a runtime lib call name for each entry in the enum RTLIB:Libcall from include/llvm/CodeGen/RuntimeLibcalls.h.
This patch added entries to the enum, but didn’t add entries to the RuntimeLibcallNames array. In fact, it looks like there are a couple of arrays in that WebAssembly file that are expecting to be changed/updated whenever the RTLIB::Libcall enum has been changed/updated. Mirrored structures like this are kind of dangerous for exactly this reason, aren’t they?
What’s the suggested way forward? (Still kind of new to LLVM) Should I try to add the corresponding entries to the WebAssembly arrays? Does someone with some familiarity with those tables want to tackle it?
-Daniel
On Jul 12, 2017, at 11:37 AM, David Blaikie <dblaikie at gmail.com<mailto:dblaikie at gmail.com>> wrote:
Reverting the patch locally does seem to remove the failure.
Here's the crash dump from one of the failing tests:
/usr/local/google/home/blaikie/dev/llvm/build/default/./bin/llc < /usr/local/google/home/blaikie/dev/llvm/src/test/CodeGen/WebAssembly/global.ll -asm-verbose=false -disable-wasm-fallthrough-return-opt -disable-wasm-explicit-locals | /usr/local/google/home/blaikie/dev/llvm/build/default/./bin/FileCheck /usr/local/google/home/blaikie/dev/llvm/src/test/CodeGen/WebAssembly/global.ll
--
Exit Code: 2
Command Output (stderr):
--
#0 0x000000000471fcb9 llvm::sys::PrintStackTrace(llvm::raw_ostream&) src/lib/Support/Unix/Signals.inc:398:11
#1 0x000000000471fe69 PrintStackTraceSignalHandler(void*) src/lib/Support/Unix/Signals.inc:462:1
#2 0x000000000471e4c3 llvm::sys::RunSignalHandlers() src/lib/Support/Signals.cpp:0:5
#3 0x00000000047201c4 SignalHandler(int) src/lib/Support/Unix/Signals.inc:252:1
#4 0x00007f770219a330 __restore_rt (/lib/x86_64-linux-gnu/libpthread.so.0+0x10330)
#5 0x00007f7700d7079a /build/eglibc-SvCtMH/eglibc-2.19/string/../sysdeps/x86_64/multiarch/strcmp-sse2-unaligned.S:29:0
#6 0x000000000350622b llvm::GetSignature(llvm::WebAssemblySubtarget const&, char const*, llvm::SmallVectorImpl<llvm::wasm::ValType>&, llvm::SmallVectorImpl<llvm::wasm::ValType>&) src/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp:1294:3
#7 0x0000000003501337 llvm::WebAssemblyMCInstLower::GetExternalSymbolSymbol(llvm::MachineOperand const&) const src/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp:100:3
#8 0x0000000003501ce5 llvm::WebAssemblyMCInstLower::Lower(llvm::MachineInstr const*, llvm::MCInst&) const src/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp:229:33
#9 0x00000000034fee64 llvm::WebAssemblyAsmPrinter::EmitInstruction(llvm::MachineInstr const*) src/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp:208:5
#10 0x00000000037a9dd5 llvm::AsmPrinter::EmitFunctionBody() src/lib/CodeGen/AsmPrinter/AsmPrinter.cpp:1037:11
#11 0x00000000021225e8 llvm::AsmPrinter::runOnMachineFunction(llvm::MachineFunction&) src/include/llvm/CodeGen/AsmPrinter.h:277:5
#12 0x00000000034ff8d3 llvm::WebAssemblyAsmPrinter::runOnMachineFunction(llvm::MachineFunction&) src/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h:49:5
#13 0x0000000003a21f61 llvm::MachineFunctionPass::runOnFunction(llvm::Function&) src/lib/CodeGen/MachineFunctionPass.cpp:62:8
#14 0x0000000003e5d92f llvm::FPPassManager::runOnFunction(llvm::Function&) src/lib/IR/LegacyPassManager.cpp:1519:27
#15 0x0000000003e5dc45 llvm::FPPassManager::runOnModule(llvm::Module&) src/lib/IR/LegacyPassManager.cpp:1540:16
#16 0x0000000003e5e3da (anonymous namespace)::MPPassManager::runOnModule(llvm::Module&) src/lib/IR/LegacyPassManager.cpp:1596:27
#17 0x0000000003e5df06 llvm::legacy::PassManagerImpl::run(llvm::Module&) src/lib/IR/LegacyPassManager.cpp:1699:16
#18 0x0000000003e5e8f1 llvm::legacy::PassManager::run(llvm::Module&) src/lib/IR/LegacyPassManager.cpp:1730:3
#19 0x00000000020ea715 compileModule(char**, llvm::LLVMContext&) src/tools/llc/llc.cpp:619:42
#20 0x00000000020e883c main src/tools/llc/llc.cpp:358:13
#21 0x00007f7700cf7f45 __libc_start_main build/eglibc-SvCtMH/eglibc-2.19/csu/libc-start.c:321:0
#22 0x00000000020e8029 _start (build/default/./bin/llc+0x20e8029)
Stack dump:
0. Program arguments: build/default/./bin/llc -asm-verbose=false -disable-wasm-fallthrough-return-opt -disable-wasm-explicit-locals
1. Running pass 'Function Pass Manager' on module '<stdin>'.
2. Running pass 'WebAssembly Assembly Printer' on function '@call_memcpy'
FileCheck error: '-' is empty.
FileCheck command line: build/default/./bin/FileCheck src/test/CodeGen/WebAssembly/global.ll
On Wed, Jul 12, 2017 at 11:31 AM Daniel Neilson <dneilson at azul.com<mailto:dneilson at azul.com>> wrote:
That’s surprising to me; this should basically be NFC since nothing is actually using the introduced intrinsic. Any information that you can share would be appreciated.
-Daniel
On Jul 12, 2017, at 11:26 AM, David Blaikie <dblaikie at gmail.com<mailto:dblaikie at gmail.com>> wrote:
This seems to be breaking internal buildbots on WebAssembly tests (test/CodeGen/WebAssembli/{byval,f16,f32,f64,frem,global,i128,legalize,mem-intrinsics}.ll
I've not reproduced this myself, but working on that now to get more info, including stack traces, etc.
On Wed, Jul 12, 2017 at 8:25 AM Daniel Neilson via llvm-commits <llvm-commits at lists.llvm.org<mailto:llvm-commits at lists.llvm.org>> wrote:
Author: dneilson
Date: Wed Jul 12 08:25:26 2017
New Revision: 307796
URL: http://llvm.org/viewvc/llvm-project?rev=307796&view=rev
Log:
Add element atomic memmove intrinsic
Summary: Continuing the work from https://reviews.llvm.org/D33240, this change introduces an element unordered-atomic memmove intrinsic. This intrinsic is essentially memmove with the implementation requirement that all loads/stores used for the copy are done with unordered-atomic loads/stores of a given element size.
Reviewers: eli.friedman, reames, mkazantsev, skatkov
Reviewed By: reames
Subscribers: llvm-commits
Differential Revision: https://reviews.llvm.org/D34884
Modified:
llvm/trunk/docs/LangRef.rst
llvm/trunk/include/llvm/CodeGen/RuntimeLibcalls.h
llvm/trunk/include/llvm/IR/IntrinsicInst.h
llvm/trunk/include/llvm/IR/Intrinsics.td
llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp
llvm/trunk/lib/IR/Verifier.cpp
llvm/trunk/test/CodeGen/X86/element-wise-atomic-memory-intrinsics.ll
llvm/trunk/test/Verifier/element-wise-atomic-memory-intrinsics.ll
Modified: llvm/trunk/docs/LangRef.rst
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/docs/LangRef.rst?rev=307796&r1=307795&r2=307796&view=diff
==============================================================================
--- llvm/trunk/docs/LangRef.rst (original)
+++ llvm/trunk/docs/LangRef.rst Wed Jul 12 08:25:26 2017
@@ -10282,6 +10282,8 @@ overlap. It copies "len" bytes of memory
to be aligned to some boundary, this can be specified as the fourth
argument, otherwise it should be set to 0 or 1 (both meaning no alignment).
+.. _int_memmove:
+
'``llvm.memmove``' Intrinsic
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -14178,4 +14180,81 @@ In the most general case call to the '``
lowered to a call to the symbol ``__llvm_memcpy_element_unordered_atomic_*``. Where '*'
is replaced with an actual element size.
+Optimizer is allowed to inline memory copy when it's profitable to do so.
+
+'``llvm.memmove.element.unordered.atomic``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+This is an overloaded intrinsic. You can use
+``llvm.memmove.element.unordered.atomic`` on any integer bit width and for
+different address spaces. Not all targets support all bit widths however.
+
+::
+
+ declare void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* <dest>,
+ i8* <src>,
+ i32 <len>,
+ i32 <element_size>)
+ declare void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* <dest>,
+ i8* <src>,
+ i64 <len>,
+ i32 <element_size>)
+
+Overview:
+"""""""""
+
+The '``llvm.memmove.element.unordered.atomic.*``' intrinsic is a specialization
+of the '``llvm.memmove.*``' intrinsic. It differs in that the ``dest`` and
+``src`` are treated as arrays with elements that are exactly ``element_size``
+bytes, and the copy between buffers uses a sequence of
+:ref:`unordered atomic <ordering>` load/store operations that are a positive
+integer multiple of the ``element_size`` in size.
+
+Arguments:
+""""""""""
+
+The first three arguments are the same as they are in the
+:ref:`@llvm.memmove <int_memmove>` intrinsic, with the added constraint that
+``len`` is required to be a positive integer multiple of the ``element_size``.
+If ``len`` is not a positive integer multiple of ``element_size``, then the
+behaviour of the intrinsic is undefined.
+
+``element_size`` must be a compile-time constant positive power of two no
+greater than a target-specific atomic access size limit.
+
+For each of the input pointers the ``align`` parameter attribute must be
+specified. It must be a power of two no less than the ``element_size``. Caller
+guarantees that both the source and destination pointers are aligned to that
+boundary.
+
+Semantics:
+""""""""""
+
+The '``llvm.memmove.element.unordered.atomic.*``' intrinsic copies ``len`` bytes
+of memory from the source location to the destination location. These locations
+are allowed to overlap. The memory copy is performed as a sequence of load/store
+operations where each access is guaranteed to be a multiple of ``element_size``
+bytes wide and aligned at an ``element_size`` boundary.
+
+The order of the copy is unspecified. The same value may be read from the source
+buffer many times, but only one write is issued to the destination buffer per
+element. It is well defined to have concurrent reads and writes to both source
+and destination provided those reads and writes are unordered atomic when
+specified.
+
+This intrinsic does not provide any additional ordering guarantees over those
+provided by a set of unordered loads from the source location and stores to the
+destination.
+
+Lowering:
+"""""""""
+
+In the most general case call to the
+'``llvm.memmove.element.unordered.atomic.*``' is lowered to a call to the symbol
+``__llvm_memmove_element_unordered_atomic_*``. Where '*' is replaced with an
+actual element size.
+
The optimizer is allowed to inline the memory copy when it's profitable to do so.
Modified: llvm/trunk/include/llvm/CodeGen/RuntimeLibcalls.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/RuntimeLibcalls.h?rev=307796&r1=307795&r2=307796&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/RuntimeLibcalls.h (original)
+++ llvm/trunk/include/llvm/CodeGen/RuntimeLibcalls.h Wed Jul 12 08:25:26 2017
@@ -340,6 +340,12 @@ namespace RTLIB {
MEMCPY_ELEMENT_UNORDERED_ATOMIC_8,
MEMCPY_ELEMENT_UNORDERED_ATOMIC_16,
+ MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1,
+ MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2,
+ MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4,
+ MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8,
+ MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16,
+
// EXCEPTION HANDLING
UNWIND_RESUME,
@@ -515,6 +521,11 @@ namespace RTLIB {
/// MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
/// UNKNOW_LIBCALL if there is none.
Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
+
+ /// getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return
+ /// MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
+ /// UNKNOW_LIBCALL if there is none.
+ Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize);
}
}
Modified: llvm/trunk/include/llvm/IR/IntrinsicInst.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/IR/IntrinsicInst.h?rev=307796&r1=307795&r2=307796&view=diff
==============================================================================
--- llvm/trunk/include/llvm/IR/IntrinsicInst.h (original)
+++ llvm/trunk/include/llvm/IR/IntrinsicInst.h Wed Jul 12 08:25:26 2017
@@ -296,6 +296,95 @@ namespace llvm {
}
};
+ class ElementUnorderedAtomicMemMoveInst : public IntrinsicInst {
+ private:
+ enum { ARG_DEST = 0, ARG_SOURCE = 1, ARG_LENGTH = 2, ARG_ELEMENTSIZE = 3 };
+
+ public:
+ Value *getRawDest() const {
+ return const_cast<Value *>(getArgOperand(ARG_DEST));
+ }
+ const Use &getRawDestUse() const { return getArgOperandUse(ARG_DEST); }
+ Use &getRawDestUse() { return getArgOperandUse(ARG_DEST); }
+
+ /// Return the arguments to the instruction.
+ Value *getRawSource() const {
+ return const_cast<Value *>(getArgOperand(ARG_SOURCE));
+ }
+ const Use &getRawSourceUse() const { return getArgOperandUse(ARG_SOURCE); }
+ Use &getRawSourceUse() { return getArgOperandUse(ARG_SOURCE); }
+
+ Value *getLength() const {
+ return const_cast<Value *>(getArgOperand(ARG_LENGTH));
+ }
+ const Use &getLengthUse() const { return getArgOperandUse(ARG_LENGTH); }
+ Use &getLengthUse() { return getArgOperandUse(ARG_LENGTH); }
+
+ bool isVolatile() const { return false; }
+
+ Value *getRawElementSizeInBytes() const {
+ return const_cast<Value *>(getArgOperand(ARG_ELEMENTSIZE));
+ }
+
+ ConstantInt *getElementSizeInBytesCst() const {
+ return cast<ConstantInt>(getRawElementSizeInBytes());
+ }
+
+ uint32_t getElementSizeInBytes() const {
+ return getElementSizeInBytesCst()->getZExtValue();
+ }
+
+ /// This is just like getRawDest, but it strips off any cast
+ /// instructions that feed it, giving the original input. The returned
+ /// value is guaranteed to be a pointer.
+ Value *getDest() const { return getRawDest()->stripPointerCasts(); }
+
+ /// This is just like getRawSource, but it strips off any cast
+ /// instructions that feed it, giving the original input. The returned
+ /// value is guaranteed to be a pointer.
+ Value *getSource() const { return getRawSource()->stripPointerCasts(); }
+
+ unsigned getDestAddressSpace() const {
+ return cast<PointerType>(getRawDest()->getType())->getAddressSpace();
+ }
+
+ unsigned getSourceAddressSpace() const {
+ return cast<PointerType>(getRawSource()->getType())->getAddressSpace();
+ }
+
+ /// Set the specified arguments of the instruction.
+ void setDest(Value *Ptr) {
+ assert(getRawDest()->getType() == Ptr->getType() &&
+ "setDest called with pointer of wrong type!");
+ setArgOperand(ARG_DEST, Ptr);
+ }
+
+ void setSource(Value *Ptr) {
+ assert(getRawSource()->getType() == Ptr->getType() &&
+ "setSource called with pointer of wrong type!");
+ setArgOperand(ARG_SOURCE, Ptr);
+ }
+
+ void setLength(Value *L) {
+ assert(getLength()->getType() == L->getType() &&
+ "setLength called with value of wrong type!");
+ setArgOperand(ARG_LENGTH, L);
+ }
+
+ void setElementSizeInBytes(Constant *V) {
+ assert(V->getType() == Type::getInt8Ty(getContext()) &&
+ "setElementSizeInBytes called with value of wrong type!");
+ setArgOperand(ARG_ELEMENTSIZE, V);
+ }
+
+ static inline bool classof(const IntrinsicInst *I) {
+ return I->getIntrinsicID() == Intrinsic::memmove_element_unordered_atomic;
+ }
+ static inline bool classof(const Value *V) {
+ return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
+ }
+ };
+
/// This is the common base class for memset/memcpy/memmove.
class MemIntrinsic : public IntrinsicInst {
public:
Modified: llvm/trunk/include/llvm/IR/Intrinsics.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/IR/Intrinsics.td?rev=307796&r1=307795&r2=307796&view=diff
==============================================================================
--- llvm/trunk/include/llvm/IR/Intrinsics.td (original)
+++ llvm/trunk/include/llvm/IR/Intrinsics.td Wed Jul 12 08:25:26 2017
@@ -873,6 +873,18 @@ def int_memcpy_element_unordered_atomic
ReadOnly<1>
]>;
+// @llvm.memmove.element.unordered.atomic.*(dest, src, length, elementsize)
+def int_memmove_element_unordered_atomic
+ : Intrinsic<[],
+ [
+ llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty, llvm_i32_ty
+ ],
+ [
+ IntrArgMemOnly, NoCapture<0>, NoCapture<1>, WriteOnly<0>,
+ ReadOnly<1>
+ ]>;
+
+
//===------------------------ Reduction Intrinsics ------------------------===//
//
def int_experimental_vector_reduce_fadd : Intrinsic<[llvm_anyfloat_ty],
Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp?rev=307796&r1=307795&r2=307796&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp Wed Jul 12 08:25:26 2017
@@ -4994,6 +4994,44 @@ SelectionDAGBuilder::visitIntrinsicCall(
DAG.setRoot(CallResult.second);
return nullptr;
}
+ case Intrinsic::memmove_element_unordered_atomic: {
+ auto &MI = cast<ElementUnorderedAtomicMemMoveInst>(I);
+ SDValue Dst = getValue(MI.getRawDest());
+ SDValue Src = getValue(MI.getRawSource());
+ SDValue Length = getValue(MI.getLength());
+
+ // Emit a library call.
+ TargetLowering::ArgListTy Args;
+ TargetLowering::ArgListEntry Entry;
+ Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
+ Entry.Node = Dst;
+ Args.push_back(Entry);
+
+ Entry.Node = Src;
+ Args.push_back(Entry);
+
+ Entry.Ty = MI.getLength()->getType();
+ Entry.Node = Length;
+ Args.push_back(Entry);
+
+ uint64_t ElementSizeConstant = MI.getElementSizeInBytes();
+ RTLIB::Libcall LibraryCall =
+ RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSizeConstant);
+ if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
+ report_fatal_error("Unsupported element size");
+
+ TargetLowering::CallLoweringInfo CLI(DAG);
+ CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
+ TLI.getLibcallCallingConv(LibraryCall),
+ Type::getVoidTy(*DAG.getContext()),
+ DAG.getExternalSymbol(TLI.getLibcallName(LibraryCall),
+ TLI.getPointerTy(DAG.getDataLayout())),
+ std::move(Args));
+
+ std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
+ DAG.setRoot(CallResult.second);
+ return nullptr;
+ }
case Intrinsic::dbg_declare: {
const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
DILocalVariable *Variable = DI.getVariable();
Modified: llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp?rev=307796&r1=307795&r2=307796&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp (original)
+++ llvm/trunk/lib/CodeGen/TargetLoweringBase.cpp Wed Jul 12 08:25:26 2017
@@ -384,6 +384,16 @@ static void InitLibcallNames(const char
"__llvm_memcpy_element_unordered_atomic_8";
Names[RTLIB::MEMCPY_ELEMENT_UNORDERED_ATOMIC_16] =
"__llvm_memcpy_element_unordered_atomic_16";
+ Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1] =
+ "__llvm_memmove_element_unordered_atomic_1";
+ Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2] =
+ "__llvm_memmove_element_unordered_atomic_2";
+ Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4] =
+ "__llvm_memmove_element_unordered_atomic_4";
+ Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8] =
+ "__llvm_memmove_element_unordered_atomic_8";
+ Names[RTLIB::MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16] =
+ "__llvm_memmove_element_unordered_atomic_16";
Names[RTLIB::UNWIND_RESUME] = "_Unwind_Resume";
Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_1] = "__sync_val_compare_and_swap_1";
Names[RTLIB::SYNC_VAL_COMPARE_AND_SWAP_2] = "__sync_val_compare_and_swap_2";
@@ -801,6 +811,23 @@ RTLIB::Libcall RTLIB::getMEMCPY_ELEMENT_
default:
return UNKNOWN_LIBCALL;
}
+}
+
+RTLIB::Libcall RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {
+ switch (ElementSize) {
+ case 1:
+ return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1;
+ case 2:
+ return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2;
+ case 4:
+ return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4;
+ case 8:
+ return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8;
+ case 16:
+ return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16;
+ default:
+ return UNKNOWN_LIBCALL;
+ }
}
/// InitCmpLibcallCCs - Set default comparison libcall CC.
Modified: llvm/trunk/lib/IR/Verifier.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/IR/Verifier.cpp?rev=307796&r1=307795&r2=307796&view=diff
==============================================================================
--- llvm/trunk/lib/IR/Verifier.cpp (original)
+++ llvm/trunk/lib/IR/Verifier.cpp Wed Jul 12 08:25:26 2017
@@ -4044,6 +4044,42 @@ void Verifier::visitIntrinsicCallSite(In
"incorrect alignment of the source argument", CS);
break;
}
+ case Intrinsic::memmove_element_unordered_atomic: {
+ auto *MI = cast<ElementUnorderedAtomicMemMoveInst>(CS.getInstruction());
+
+ ConstantInt *ElementSizeCI =
+ dyn_cast<ConstantInt>(MI->getRawElementSizeInBytes());
+ Assert(ElementSizeCI,
+ "element size of the element-wise unordered atomic memory "
+ "intrinsic must be a constant int",
+ CS);
+ const APInt &ElementSizeVal = ElementSizeCI->getValue();
+ Assert(ElementSizeVal.isPowerOf2(),
+ "element size of the element-wise atomic memory intrinsic "
+ "must be a power of 2",
+ CS);
+
+ if (auto *LengthCI = dyn_cast<ConstantInt>(MI->getLength())) {
+ uint64_t Length = LengthCI->getZExtValue();
+ uint64_t ElementSize = MI->getElementSizeInBytes();
+ Assert((Length % ElementSize) == 0,
+ "constant length must be a multiple of the element size in the "
+ "element-wise atomic memory intrinsic",
+ CS);
+ }
+
+ auto IsValidAlignment = [&](uint64_t Alignment) {
+ return isPowerOf2_64(Alignment) && ElementSizeVal.ule(Alignment);
+ };
+ uint64_t DstAlignment = CS.getParamAlignment(0),
+ SrcAlignment = CS.getParamAlignment(1);
+ Assert(IsValidAlignment(DstAlignment),
+ "incorrect alignment of the destination argument", CS);
+ Assert(IsValidAlignment(SrcAlignment),
+ "incorrect alignment of the source argument", CS);
+
+ break;
+ }
case Intrinsic::gcroot:
case Intrinsic::gcwrite:
case Intrinsic::gcread:
Modified: llvm/trunk/test/CodeGen/X86/element-wise-atomic-memory-intrinsics.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/element-wise-atomic-memory-intrinsics.ll?rev=307796&r1=307795&r2=307796&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/X86/element-wise-atomic-memory-intrinsics.ll (original)
+++ llvm/trunk/test/CodeGen/X86/element-wise-atomic-memory-intrinsics.ll Wed Jul 12 08:25:26 2017
@@ -62,4 +62,67 @@ define void @test_memcpy_args(i8** %Stor
call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %Dst, i8* align 4 %Src, i32 4, i32 4) ret void
}
+define i8* @test_memmove1(i8* %P, i8* %Q) {
+ ; CHECK: test_memmove
+ call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* align 4 %Q, i32 1, i32 1)
+ ret i8* %P
+ ; 3rd arg (%edx) -- length
+ ; CHECK-DAG: movl $1, %edx
+ ; CHECK: __llvm_memmove_element_unordered_atomic_1
+}
+
+define i8* @test_memmove2(i8* %P, i8* %Q) {
+ ; CHECK: test_memmove2
+ call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* align 4 %Q, i32 2, i32 2)
+ ret i8* %P
+ ; 3rd arg (%edx) -- length
+ ; CHECK-DAG: movl $2, %edx
+ ; CHECK: __llvm_memmove_element_unordered_atomic_2
+}
+
+define i8* @test_memmove4(i8* %P, i8* %Q) {
+ ; CHECK: test_memmove4
+ call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* align 4 %Q, i32 4, i32 4)
+ ret i8* %P
+ ; 3rd arg (%edx) -- length
+ ; CHECK-DAG: movl $4, %edx
+ ; CHECK: __llvm_memmove_element_unordered_atomic_4
+}
+
+define i8* @test_memmove8(i8* %P, i8* %Q) {
+ ; CHECK: test_memmove8
+ call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %P, i8* align 8 %Q, i32 8, i32 8)
+ ret i8* %P
+ ; 3rd arg (%edx) -- length
+ ; CHECK-DAG: movl $8, %edx
+ ; CHECK: __llvm_memmove_element_unordered_atomic_8
+}
+
+define i8* @test_memmove16(i8* %P, i8* %Q) {
+ ; CHECK: test_memmove16
+ call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %P, i8* align 16 %Q, i32 16, i32 16)
+ ret i8* %P
+ ; 3rd arg (%edx) -- length
+ ; CHECK-DAG: movl $16, %edx
+ ; CHECK: __llvm_memmove_element_unordered_atomic_16
+}
+
+define void @test_memmove_args(i8** %Storage) {
+ ; CHECK: test_memmove_args
+ %Dst = load i8*, i8** %Storage
+ %Src.addr = getelementptr i8*, i8** %Storage, i64 1
+ %Src = load i8*, i8** %Src.addr
+
+ ; 1st arg (%rdi)
+ ; CHECK-DAG: movq (%rdi), [[REG1:%r.+]]
+ ; CHECK-DAG: movq [[REG1]], %rdi
+ ; 2nd arg (%rsi)
+ ; CHECK-DAG: movq 8(%rdi), %rsi
+ ; 3rd arg (%edx) -- length
+ ; CHECK-DAG: movl $4, %edx
+ ; CHECK: __llvm_memmove_element_unordered_atomic_4
+ call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %Dst, i8* align 4 %Src, i32 4, i32 4) ret void
+}
+
declare void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32) nounwind
+declare void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32) nounwind
Modified: llvm/trunk/test/Verifier/element-wise-atomic-memory-intrinsics.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Verifier/element-wise-atomic-memory-intrinsics.ll?rev=307796&r1=307795&r2=307796&view=diff
==============================================================================
--- llvm/trunk/test/Verifier/element-wise-atomic-memory-intrinsics.ll (original)
+++ llvm/trunk/test/Verifier/element-wise-atomic-memory-intrinsics.ll Wed Jul 12 08:25:26 2017
@@ -22,4 +22,28 @@ define void @test_memcpy(i8* %P, i8* %Q,
ret void
}
declare void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32) nounwind
+
+define void @test_memmove(i8* %P, i8* %Q, i32 %A, i32 %E) {
+ ; CHECK: element size of the element-wise unordered atomic memory intrinsic must be a constant int
+ call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* align 4 %Q, i32 1, i32 %E)
+ ; CHECK: element size of the element-wise atomic memory intrinsic must be a power of 2
+ call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* align 4 %Q, i32 1, i32 3)
+
+ ; CHECK: constant length must be a multiple of the element size in the element-wise atomic memory intrinsic
+ call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* align 4 %Q, i32 7, i32 4)
+
+ ; CHECK: incorrect alignment of the destination argument
+ call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* %P, i8* align 4 %Q, i32 1, i32 1)
+ ; CHECK: incorrect alignment of the destination argument
+ call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %P, i8* align 4 %Q, i32 4, i32 4)
+
+ ; CHECK: incorrect alignment of the source argument
+ call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* %Q, i32 1, i32 1)
+ ; CHECK: incorrect alignment of the source argument
+ call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %P, i8* align 1 %Q, i32 4, i32 4)
+
+ ret void
+}
+declare void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32) nounwind
+
; CHECK: input module is broken!
_______________________________________________
llvm-commits mailing list
llvm-commits at lists.llvm.org<mailto:llvm-commits at lists.llvm.org>
http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20170712/1c3e5951/attachment.html>
More information about the llvm-commits
mailing list