[llvm] b60a9cc - AtomicExpand: Use InstSimplifyFolder

Matt Arsenault via llvm-commits llvm-commits at lists.llvm.org
Mon Oct 31 23:31:48 PDT 2022 

Author: Matt Arsenault
Date: 2022-10-31T23:31:42-07:00
New Revision: b60a9ccd02a233b42f001fb17b988ba5cde1c7cf

URL: https://github.com/llvm/llvm-project/commit/b60a9ccd02a233b42f001fb17b988ba5cde1c7cf
DIFF: https://github.com/llvm/llvm-project/commit/b60a9ccd02a233b42f001fb17b988ba5cde1c7cf.diff

LOG: AtomicExpand: Use InstSimplifyFolder

Automatically cleanup operations if we know the atomic has higher
alignment.

Added: 
    

Modified: 
    llvm/include/llvm/CodeGen/AtomicExpandUtils.h
    llvm/lib/CodeGen/AtomicExpandPass.cpp
    llvm/test/CodeGen/ARM/atomic-op.ll
    llvm/test/Transforms/AtomicExpand/AArch64/pcsections.ll
    llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
    llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8.ll
    llvm/test/Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll
    llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll

Removed: 
    


################################################################################
diff  --git a/llvm/include/llvm/CodeGen/AtomicExpandUtils.h b/llvm/include/llvm/CodeGen/AtomicExpandUtils.h
index 7615ddb0ab3d2..1cb410a0c31c6 100644
--- a/llvm/include/llvm/CodeGen/AtomicExpandUtils.h
+++ b/llvm/include/llvm/CodeGen/AtomicExpandUtils.h
@@ -22,7 +22,7 @@ class Value;
 /// (the builder, %addr, %loaded, %new_val, ordering,
 ///  /* OUT */ %success, /* OUT */ %new_loaded)
 using CreateCmpXchgInstFun =
-    function_ref<void(IRBuilder<> &, Value *, Value *, Value *, Align,
+    function_ref<void(IRBuilderBase &, Value *, Value *, Value *, Align,
                       AtomicOrdering, SyncScope::ID, Value *&, Value *&)>;
 
 /// Expand an atomic RMW instruction into a loop utilizing

diff  --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 76ea1b81ef175..8b46e9580729a 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -17,6 +17,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/InstSimplifyFolder.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/AtomicExpandUtils.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
@@ -60,6 +61,7 @@ namespace {
 
 class AtomicExpand : public FunctionPass {
   const TargetLowering *TLI = nullptr;
+  const DataLayout *DL = nullptr;
 
 public:
   static char ID; // Pass identification, replacement for typeid
@@ -83,13 +85,13 @@ class AtomicExpand : public FunctionPass {
   bool tryExpandAtomicRMW(AtomicRMWInst *AI);
   AtomicRMWInst *convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI);
   Value *
-  insertRMWLLSCLoop(IRBuilder<> &Builder, Type *ResultTy, Value *Addr,
+  insertRMWLLSCLoop(IRBuilderBase &Builder, Type *ResultTy, Value *Addr,
                     Align AddrAlign, AtomicOrdering MemOpOrder,
-                    function_ref<Value *(IRBuilder<> &, Value *)> PerformOp);
-  void
-  expandAtomicOpToLLSC(Instruction *I, Type *ResultTy, Value *Addr,
-                       Align AddrAlign, AtomicOrdering MemOpOrder,
-                       function_ref<Value *(IRBuilder<> &, Value *)> PerformOp);
+                    function_ref<Value *(IRBuilderBase &, Value *)> PerformOp);
+  void expandAtomicOpToLLSC(
+      Instruction *I, Type *ResultTy, Value *Addr, Align AddrAlign,
+      AtomicOrdering MemOpOrder,
+      function_ref<Value *(IRBuilderBase &, Value *)> PerformOp);
   void expandPartwordAtomicRMW(
       AtomicRMWInst *I, TargetLoweringBase::AtomicExpansionKind ExpansionKind);
   AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
@@ -98,12 +100,11 @@ class AtomicExpand : public FunctionPass {
   void expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI);
 
   AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI);
-  static Value *
-  insertRMWCmpXchgLoop(IRBuilder<> &Builder, Type *ResultType, Value *Addr,
-                       Align AddrAlign, AtomicOrdering MemOpOrder,
-                       SyncScope::ID SSID,
-                       function_ref<Value *(IRBuilder<> &, Value *)> PerformOp,
-                       CreateCmpXchgInstFun CreateCmpXchg);
+  static Value *insertRMWCmpXchgLoop(
+      IRBuilderBase &Builder, Type *ResultType, Value *Addr, Align AddrAlign,
+      AtomicOrdering MemOpOrder, SyncScope::ID SSID,
+      function_ref<Value *(IRBuilderBase &, Value *)> PerformOp,
+      CreateCmpXchgInstFun CreateCmpXchg);
   bool tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI);
 
   bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI);
@@ -126,9 +127,11 @@ class AtomicExpand : public FunctionPass {
 };
 
 // IRBuilder to be used for replacement atomic instructions.
-struct ReplacementIRBuilder : IRBuilder<> {
+struct ReplacementIRBuilder : IRBuilder<InstSimplifyFolder> {
   // Preserves the DebugLoc from I, and preserves still valid metadata.
-  explicit ReplacementIRBuilder(Instruction *I) : IRBuilder<>(I) {
+  explicit ReplacementIRBuilder(Instruction *I, const DataLayout &DL)
+      : IRBuilder(I->getContext(), DL) {
+    SetInsertPoint(I);
     this->CollectMetadataToCopy(I, {LLVMContext::MD_pcsections});
   }
 };
@@ -182,9 +185,11 @@ bool AtomicExpand::runOnFunction(Function &F) {
     return false;
 
   auto &TM = TPC->getTM<TargetMachine>();
-  if (!TM.getSubtargetImpl(F)->enableAtomicExpand())
+  const auto *Subtarget = TM.getSubtargetImpl(F);
+  if (!Subtarget->enableAtomicExpand())
     return false;
-  TLI = TM.getSubtargetImpl(F)->getTargetLowering();
+  TLI = Subtarget->getTargetLowering();
+  DL = &F.getParent()->getDataLayout();
 
   SmallVector<Instruction *, 1> AtomicInsts;
 
@@ -318,7 +323,7 @@ bool AtomicExpand::runOnFunction(Function &F) {
 }
 
 bool AtomicExpand::bracketInstWithFences(Instruction *I, AtomicOrdering Order) {
-  ReplacementIRBuilder Builder(I);
+  ReplacementIRBuilder Builder(I, *DL);
 
   auto LeadingFence = TLI->emitLeadingFence(Builder, I, Order);
 
@@ -347,7 +352,7 @@ LoadInst *AtomicExpand::convertAtomicLoadToIntegerType(LoadInst *LI) {
   auto *M = LI->getModule();
   Type *NewTy = getCorrespondingIntegerType(LI->getType(), M->getDataLayout());
 
-  ReplacementIRBuilder Builder(LI);
+  ReplacementIRBuilder Builder(LI, *DL);
 
   Value *Addr = LI->getPointerOperand();
   Type *PT = PointerType::get(NewTy, Addr->getType()->getPointerAddressSpace());
@@ -371,7 +376,7 @@ AtomicExpand::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
   Type *NewTy =
       getCorrespondingIntegerType(RMWI->getType(), M->getDataLayout());
 
-  ReplacementIRBuilder Builder(RMWI);
+  ReplacementIRBuilder Builder(RMWI, *DL);
 
   Value *Addr = RMWI->getPointerOperand();
   Value *Val = RMWI->getValOperand();
@@ -403,7 +408,7 @@ bool AtomicExpand::tryExpandAtomicLoad(LoadInst *LI) {
     expandAtomicOpToLLSC(
         LI, LI->getType(), LI->getPointerOperand(), LI->getAlign(),
         LI->getOrdering(),
-        [](IRBuilder<> &Builder, Value *Loaded) { return Loaded; });
+        [](IRBuilderBase &Builder, Value *Loaded) { return Loaded; });
     return true;
   case TargetLoweringBase::AtomicExpansionKind::LLOnly:
     return expandAtomicLoadToLL(LI);
@@ -433,7 +438,7 @@ bool AtomicExpand::tryExpandAtomicStore(StoreInst *SI) {
 }
 
 bool AtomicExpand::expandAtomicLoadToLL(LoadInst *LI) {
-  ReplacementIRBuilder Builder(LI);
+  ReplacementIRBuilder Builder(LI, *DL);
 
   // On some architectures, load-linked instructions are atomic for larger
   // sizes than normal loads. For example, the only 64-bit load guaranteed
@@ -449,7 +454,7 @@ bool AtomicExpand::expandAtomicLoadToLL(LoadInst *LI) {
 }
 
 bool AtomicExpand::expandAtomicLoadToCmpXchg(LoadInst *LI) {
-  ReplacementIRBuilder Builder(LI);
+  ReplacementIRBuilder Builder(LI, *DL);
   AtomicOrdering Order = LI->getOrdering();
   if (Order == AtomicOrdering::Unordered)
     Order = AtomicOrdering::Monotonic;
@@ -478,7 +483,7 @@ bool AtomicExpand::expandAtomicLoadToCmpXchg(LoadInst *LI) {
 /// mechanism, we convert back to the old format which the backends understand.
 /// Each backend will need individual work to recognize the new format.
 StoreInst *AtomicExpand::convertAtomicStoreToIntegerType(StoreInst *SI) {
-  ReplacementIRBuilder Builder(SI);
+  ReplacementIRBuilder Builder(SI, *DL);
   auto *M = SI->getModule();
   Type *NewTy = getCorrespondingIntegerType(SI->getValueOperand()->getType(),
                                             M->getDataLayout());
@@ -504,7 +509,7 @@ void AtomicExpand::expandAtomicStore(StoreInst *SI) {
   // or lock cmpxchg8/16b on X86, as these are atomic for larger sizes.
   // It is the responsibility of the target to only signal expansion via
   // shouldExpandAtomicRMW in cases where this is required and possible.
-  ReplacementIRBuilder Builder(SI);
+  ReplacementIRBuilder Builder(SI, *DL);
   AtomicOrdering Ordering = SI->getOrdering();
   assert(Ordering != AtomicOrdering::NotAtomic);
   AtomicOrdering RMWOrdering = Ordering == AtomicOrdering::Unordered
@@ -519,7 +524,7 @@ void AtomicExpand::expandAtomicStore(StoreInst *SI) {
   tryExpandAtomicRMW(AI);
 }
 
-static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr,
+static void createCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr,
                                  Value *Loaded, Value *NewVal, Align AddrAlign,
                                  AtomicOrdering MemOpOrder, SyncScope::ID SSID,
                                  Value *&Success, Value *&NewLoaded) {
@@ -559,7 +564,7 @@ bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) {
       expandPartwordAtomicRMW(AI,
                               TargetLoweringBase::AtomicExpansionKind::LLSC);
     } else {
-      auto PerformOp = [&](IRBuilder<> &Builder, Value *Loaded) {
+      auto PerformOp = [&](IRBuilderBase &Builder, Value *Loaded) {
         return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded,
                                    AI->getValOperand());
       };
@@ -669,9 +674,9 @@ raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) {
 ///       include only the part that would've been loaded from Addr.
 ///
 /// Inv_Mask: The inverse of Mask.
-static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I,
-                                           Type *ValueType, Value *Addr,
-                                           Align AddrAlign,
+static PartwordMaskValues createMaskInstrs(IRBuilderBase &Builder,
+                                           Instruction *I, Type *ValueType,
+                                           Value *Addr, Align AddrAlign,
                                            unsigned MinWordSize) {
   PartwordMaskValues PMV;
 
@@ -737,7 +742,7 @@ static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I,
   return PMV;
 }
 
-static Value *extractMaskedValue(IRBuilder<> &Builder, Value *WideWord,
+static Value *extractMaskedValue(IRBuilderBase &Builder, Value *WideWord,
                                  const PartwordMaskValues &PMV) {
   assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
   if (PMV.WordType == PMV.ValueType)
@@ -748,7 +753,7 @@ static Value *extractMaskedValue(IRBuilder<> &Builder, Value *WideWord,
   return Trunc;
 }
 
-static Value *insertMaskedValue(IRBuilder<> &Builder, Value *WideWord,
+static Value *insertMaskedValue(IRBuilderBase &Builder, Value *WideWord,
                                 Value *Updated, const PartwordMaskValues &PMV) {
   assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");
   assert(Updated->getType() == PMV.ValueType && "Value type mismatch");
@@ -767,7 +772,7 @@ static Value *insertMaskedValue(IRBuilder<> &Builder, Value *WideWord,
 /// operation. (That is, only the bits under the Mask should be
 /// affected by the operation)
 static Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op,
-                                    IRBuilder<> &Builder, Value *Loaded,
+                                    IRBuilderBase &Builder, Value *Loaded,
                                     Value *Shifted_Inc, Value *Inc,
                                     const PartwordMaskValues &PMV) {
   // TODO: update to use
@@ -822,7 +827,7 @@ void AtomicExpand::expandPartwordAtomicRMW(
   AtomicOrdering MemOpOrder = AI->getOrdering();
   SyncScope::ID SSID = AI->getSyncScopeID();
 
-  ReplacementIRBuilder Builder(AI);
+  ReplacementIRBuilder Builder(AI, *DL);
 
   PartwordMaskValues PMV =
       createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
@@ -838,7 +843,7 @@ void AtomicExpand::expandPartwordAtomicRMW(
                           PMV.ShiftAmt, "ValOperand_Shifted");
   }
 
-  auto PerformPartwordOp = [&](IRBuilder<> &Builder, Value *Loaded) {
+  auto PerformPartwordOp = [&](IRBuilderBase &Builder, Value *Loaded) {
     return performMaskedAtomicOp(AI->getOperation(), Builder, Loaded,
                                  ValOperand_Shifted, AI->getValOperand(), PMV);
   };
@@ -862,7 +867,7 @@ void AtomicExpand::expandPartwordAtomicRMW(
 
 // Widen the bitwise atomicrmw (or/xor/and) to the minimum supported width.
 AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
-  ReplacementIRBuilder Builder(AI);
+  ReplacementIRBuilder Builder(AI, *DL);
   AtomicRMWInst::BinOp Op = AI->getOperation();
 
   assert((Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
@@ -937,7 +942,7 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
 
   BasicBlock *BB = CI->getParent();
   Function *F = BB->getParent();
-  ReplacementIRBuilder Builder(CI);
+  ReplacementIRBuilder Builder(CI, *DL);
   LLVMContext &Ctx = Builder.getContext();
 
   BasicBlock *EndBB =
@@ -1023,8 +1028,8 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
 void AtomicExpand::expandAtomicOpToLLSC(
     Instruction *I, Type *ResultType, Value *Addr, Align AddrAlign,
     AtomicOrdering MemOpOrder,
-    function_ref<Value *(IRBuilder<> &, Value *)> PerformOp) {
-  ReplacementIRBuilder Builder(I);
+    function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
+  ReplacementIRBuilder Builder(I, *DL);
   Value *Loaded = insertRMWLLSCLoop(Builder, ResultType, Addr, AddrAlign,
                                     MemOpOrder, PerformOp);
 
@@ -1033,7 +1038,7 @@ void AtomicExpand::expandAtomicOpToLLSC(
 }
 
 void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
-  ReplacementIRBuilder Builder(AI);
+  ReplacementIRBuilder Builder(AI, *DL);
 
   PartwordMaskValues PMV =
       createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
@@ -1059,7 +1064,7 @@ void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
 }
 
 void AtomicExpand::expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI) {
-  ReplacementIRBuilder Builder(CI);
+  ReplacementIRBuilder Builder(CI, *DL);
 
   PartwordMaskValues PMV = createMaskInstrs(
       Builder, CI, CI->getCompareOperand()->getType(), CI->getPointerOperand(),
@@ -1086,9 +1091,9 @@ void AtomicExpand::expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI) {
 }
 
 Value *AtomicExpand::insertRMWLLSCLoop(
-    IRBuilder<> &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
+    IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
     AtomicOrdering MemOpOrder,
-    function_ref<Value *(IRBuilder<> &, Value *)> PerformOp) {
+    function_ref<Value *(IRBuilderBase &, Value *)> PerformOp) {
   LLVMContext &Ctx = Builder.getContext();
   BasicBlock *BB = Builder.GetInsertBlock();
   Function *F = BB->getParent();
@@ -1146,7 +1151,7 @@ AtomicExpand::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
   Type *NewTy = getCorrespondingIntegerType(CI->getCompareOperand()->getType(),
                                             M->getDataLayout());
 
-  ReplacementIRBuilder Builder(CI);
+  ReplacementIRBuilder Builder(CI, *DL);
 
   Value *Addr = CI->getPointerOperand();
   Type *PT = PointerType::get(NewTy, Addr->getType()->getPointerAddressSpace());
@@ -1270,7 +1275,7 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
       BasicBlock::Create(Ctx, "cmpxchg.fencedstore", F, TryStoreBB);
   auto StartBB = BasicBlock::Create(Ctx, "cmpxchg.start", F, ReleasingStoreBB);
 
-  ReplacementIRBuilder Builder(CI);
+  ReplacementIRBuilder Builder(CI, *DL);
 
   // The split call above "helpfully" added a branch at the end of BB (to the
   // wrong place), but we might want a fence too. It's easiest to just remove
@@ -1450,9 +1455,9 @@ bool AtomicExpand::simplifyIdempotentRMW(AtomicRMWInst *RMWI) {
 }
 
 Value *AtomicExpand::insertRMWCmpXchgLoop(
-    IRBuilder<> &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
+    IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
     AtomicOrdering MemOpOrder, SyncScope::ID SSID,
-    function_ref<Value *(IRBuilder<> &, Value *)> PerformOp,
+    function_ref<Value *(IRBuilderBase &, Value *)> PerformOp,
     CreateCmpXchgInstFun CreateCmpXchg) {
   LLVMContext &Ctx = Builder.getContext();
   BasicBlock *BB = Builder.GetInsertBlock();
@@ -1535,11 +1540,11 @@ bool AtomicExpand::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
 // Note: This function is exposed externally by AtomicExpandUtils.h
 bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
                                     CreateCmpXchgInstFun CreateCmpXchg) {
-  ReplacementIRBuilder Builder(AI);
+  ReplacementIRBuilder Builder(AI, AI->getModule()->getDataLayout());
   Value *Loaded = AtomicExpand::insertRMWCmpXchgLoop(
       Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(),
       AI->getOrdering(), AI->getSyncScopeID(),
-      [&](IRBuilder<> &Builder, Value *Loaded) {
+      [&](IRBuilderBase &Builder, Value *Loaded) {
         return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded,
                                    AI->getValOperand());
       },
@@ -1689,7 +1694,7 @@ void AtomicExpand::expandAtomicRMWToLibcall(AtomicRMWInst *I) {
   // CAS libcall, via a CAS loop, instead.
   if (!Success) {
     expandAtomicRMWToCmpXchg(
-        I, [this](IRBuilder<> &Builder, Value *Addr, Value *Loaded,
+        I, [this](IRBuilderBase &Builder, Value *Addr, Value *Loaded,
                   Value *NewVal, Align Alignment, AtomicOrdering MemOpOrder,
                   SyncScope::ID SSID, Value *&Success, Value *&NewLoaded) {
           // Create the CAS instruction normally...

diff  --git a/llvm/test/CodeGen/ARM/atomic-op.ll b/llvm/test/CodeGen/ARM/atomic-op.ll
index be4f1868b44a4..d831c94108e40 100644
--- a/llvm/test/CodeGen/ARM/atomic-op.ll
+++ b/llvm/test/CodeGen/ARM/atomic-op.ll
@@ -6,6 +6,7 @@
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
+; CHECK-LABEL: _func:
 define void @func(i32 %argc, i8** %argv) nounwind {
 entry:
 	%argc.addr = alloca i32		; <i32*> [#uses=1]
@@ -161,8 +162,8 @@ entry:
 	store i32 %12, i32* %old
 	call void asm sideeffect "", "~{memory},~{dirflag},~{fpsr},~{flags}"()
   ; CHECK: ldrex
-  ; CHECK: cmp
   ; CHECK: strex
+  ; CHECK: cmp
   ; CHECK-T1: bl ___sync_fetch_and_umax_4
   ; CHECK-T1-M0: bl ___atomic_compare_exchange_4
   ; CHECK-BAREMETAL: cmp
@@ -171,8 +172,8 @@ entry:
 	store i32 %13, i32* %old
 	call void asm sideeffect "", "~{memory},~{dirflag},~{fpsr},~{flags}"()
   ; CHECK: ldrex
-  ; CHECK: cmp
   ; CHECK: strex
+  ; CHECK: cmp
   ; CHECK-T1: bl ___sync_fetch_and_umax_4
   ; CHECK-T1-M0: bl ___atomic_compare_exchange_4
   ; CHECK-BAREMETAL: cmp
@@ -183,14 +184,15 @@ entry:
   ret void
 }
 
+; CHECK-LABEL: _func2:
 define void @func2() nounwind {
 entry:
   %val = alloca i16
   %old = alloca i16
   store i16 31, i16* %val
   ; CHECK: ldrex
-  ; CHECK: cmp
   ; CHECK: strex
+  ; CHECK: cmp
   ; CHECK-T1: bl ___sync_fetch_and_umin_2
   ; CHECK-T1-M0: bl ___atomic_compare_exchange_2
   ; CHECK-BAREMETAL: cmp
@@ -199,8 +201,8 @@ entry:
   store i16 %0, i16* %old
   %uneg = sub i16 0, 1
   ; CHECK: ldrex
-  ; CHECK: cmp
   ; CHECK: strex
+  ; CHECK: cmp
   ; CHECK-T1: bl ___sync_fetch_and_umin_2
   ; CHECK-T1-M0: bl ___atomic_compare_exchange_2
   ; CHECK-BAREMETAL: cmp
@@ -217,8 +219,8 @@ entry:
   %2 = atomicrmw umax i16* %val, i16 1 monotonic
   store i16 %2, i16* %old
   ; CHECK: ldrex
-  ; CHECK: cmp
   ; CHECK: strex
+  ; CHECK: cmp
   ; CHECK-T1: bl ___sync_fetch_and_umax_2
   ; CHECK-T1-M0: bl ___atomic_compare_exchange_2
   ; CHECK-BAREMETAL: cmp
@@ -228,14 +230,15 @@ entry:
   ret void
 }
 
+; CHECK-LABEL: _func3:
 define void @func3() nounwind {
 entry:
   %val = alloca i8
   %old = alloca i8
   store i8 31, i8* %val
   ; CHECK: ldrex
-  ; CHECK: cmp
   ; CHECK: strex
+  ; CHECK: cmp
   ; CHECK-T1: bl ___sync_fetch_and_umin_1
   ; CHECK-T1-M0: bl ___atomic_compare_exchange_1
   ; CHECK-BAREMETAL: cmp
@@ -243,8 +246,8 @@ entry:
   %0 = atomicrmw umin i8* %val, i8 16 monotonic
   store i8 %0, i8* %old
   ; CHECK: ldrex
-  ; CHECK: cmp
   ; CHECK: strex
+  ; CHECK: cmp
   ; CHECK-T1: bl ___sync_fetch_and_umin_1
   ; CHECK-T1-M0: bl ___atomic_compare_exchange_1
   ; CHECK-BAREMETAL: cmp
@@ -253,8 +256,8 @@ entry:
   %1 = atomicrmw umin i8* %val, i8 %uneg monotonic
   store i8 %1, i8* %old
   ; CHECK: ldrex
-  ; CHECK: cmp
   ; CHECK: strex
+  ; CHECK: cmp
   ; CHECK-T1: bl ___sync_fetch_and_umax_1
   ; CHECK-T1-M0: bl ___atomic_compare_exchange_1
   ; CHECK-BAREMETAL: cmp
@@ -262,8 +265,8 @@ entry:
   %2 = atomicrmw umax i8* %val, i8 1 monotonic
   store i8 %2, i8* %old
   ; CHECK: ldrex
-  ; CHECK: cmp
   ; CHECK: strex
+  ; CHECK: cmp
   ; CHECK-T1: bl ___sync_fetch_and_umax_1
   ; CHECK-T1-M0: bl ___atomic_compare_exchange_1
   ; CHECK-BAREMETAL: cmp
@@ -273,7 +276,7 @@ entry:
   ret void
 }
 
-; CHECK: func4
+; CHECK-LABEL: _func4:
 ; This function should not need to use callee-saved registers.
 ; rdar://problem/12203728
 ; CHECK-NOT: r4

diff  --git a/llvm/test/Transforms/AtomicExpand/AArch64/pcsections.ll b/llvm/test/Transforms/AtomicExpand/AArch64/pcsections.ll
index 6d15d28509d53..36591f91827fd 100644
--- a/llvm/test/Transforms/AtomicExpand/AArch64/pcsections.ll
+++ b/llvm/test/Transforms/AtomicExpand/AArch64/pcsections.ll
@@ -115,8 +115,7 @@ define void @atomic8_add_monotonic(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] monotonic monotonic, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -135,8 +134,7 @@ define void @atomic8_sub_monotonic(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] monotonic monotonic, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -155,8 +153,7 @@ define void @atomic8_and_monotonic(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 0 monotonic monotonic, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -175,8 +172,7 @@ define void @atomic8_or_monotonic(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] monotonic monotonic, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -195,8 +191,7 @@ define void @atomic8_xor_monotonic(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] monotonic monotonic, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -215,11 +210,9 @@ define void @atomic8_nand_monotonic(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] monotonic monotonic, align 1, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 -1 monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -255,8 +248,7 @@ define void @atomic8_add_acquire(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] acquire acquire, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -275,8 +267,7 @@ define void @atomic8_sub_acquire(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] acquire acquire, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -295,8 +286,7 @@ define void @atomic8_and_acquire(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 0 acquire acquire, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -315,8 +305,7 @@ define void @atomic8_or_acquire(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] acquire acquire, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -335,8 +324,7 @@ define void @atomic8_xor_acquire(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] acquire acquire, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -355,11 +343,9 @@ define void @atomic8_nand_acquire(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acquire acquire, align 1, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 -1 acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -395,8 +381,7 @@ define void @atomic8_add_release(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] release monotonic, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -415,8 +400,7 @@ define void @atomic8_sub_release(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] release monotonic, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -435,8 +419,7 @@ define void @atomic8_and_release(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 0 release monotonic, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -455,8 +438,7 @@ define void @atomic8_or_release(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] release monotonic, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -475,8 +457,7 @@ define void @atomic8_xor_release(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] release monotonic, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -495,11 +476,9 @@ define void @atomic8_nand_release(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] release monotonic, align 1, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 -1 release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -535,8 +514,7 @@ define void @atomic8_add_acq_rel(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] acq_rel acquire, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -555,8 +533,7 @@ define void @atomic8_sub_acq_rel(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] acq_rel acquire, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -575,8 +552,7 @@ define void @atomic8_and_acq_rel(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 0 acq_rel acquire, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -595,8 +571,7 @@ define void @atomic8_or_acq_rel(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] acq_rel acquire, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -615,8 +590,7 @@ define void @atomic8_xor_acq_rel(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] acq_rel acquire, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -635,11 +609,9 @@ define void @atomic8_nand_acq_rel(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acq_rel acquire, align 1, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 -1 acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -675,8 +647,7 @@ define void @atomic8_add_seq_cst(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] seq_cst seq_cst, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -695,8 +666,7 @@ define void @atomic8_sub_seq_cst(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] seq_cst seq_cst, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -715,8 +685,7 @@ define void @atomic8_and_seq_cst(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 0 seq_cst seq_cst, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -735,8 +704,7 @@ define void @atomic8_or_seq_cst(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] seq_cst seq_cst, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -755,8 +723,7 @@ define void @atomic8_xor_seq_cst(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[LOADED]] seq_cst seq_cst, align 1, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -775,11 +742,9 @@ define void @atomic8_nand_seq_cst(i8* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] seq_cst seq_cst, align 1, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 -1 seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -978,8 +943,7 @@ define void @atomic16_add_monotonic(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] monotonic monotonic, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -998,8 +962,7 @@ define void @atomic16_sub_monotonic(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] monotonic monotonic, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1018,8 +981,7 @@ define void @atomic16_and_monotonic(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 0 monotonic monotonic, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1038,8 +1000,7 @@ define void @atomic16_or_monotonic(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] monotonic monotonic, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1058,8 +1019,7 @@ define void @atomic16_xor_monotonic(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] monotonic monotonic, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1078,11 +1038,9 @@ define void @atomic16_nand_monotonic(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] monotonic monotonic, align 2, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 -1 monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -1118,8 +1076,7 @@ define void @atomic16_add_acquire(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] acquire acquire, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1138,8 +1095,7 @@ define void @atomic16_sub_acquire(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] acquire acquire, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1158,8 +1114,7 @@ define void @atomic16_and_acquire(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 0 acquire acquire, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1178,8 +1133,7 @@ define void @atomic16_or_acquire(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] acquire acquire, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1198,8 +1152,7 @@ define void @atomic16_xor_acquire(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] acquire acquire, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1218,11 +1171,9 @@ define void @atomic16_nand_acquire(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acquire acquire, align 2, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 -1 acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -1258,8 +1209,7 @@ define void @atomic16_add_release(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] release monotonic, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1278,8 +1228,7 @@ define void @atomic16_sub_release(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] release monotonic, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1298,8 +1247,7 @@ define void @atomic16_and_release(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 0 release monotonic, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1318,8 +1266,7 @@ define void @atomic16_or_release(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] release monotonic, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1338,8 +1285,7 @@ define void @atomic16_xor_release(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] release monotonic, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1358,11 +1304,9 @@ define void @atomic16_nand_release(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] release monotonic, align 2, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 -1 release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -1398,8 +1342,7 @@ define void @atomic16_add_acq_rel(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] acq_rel acquire, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1418,8 +1361,7 @@ define void @atomic16_sub_acq_rel(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] acq_rel acquire, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1438,8 +1380,7 @@ define void @atomic16_and_acq_rel(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 0 acq_rel acquire, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1458,8 +1399,7 @@ define void @atomic16_or_acq_rel(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] acq_rel acquire, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1478,8 +1418,7 @@ define void @atomic16_xor_acq_rel(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] acq_rel acquire, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1498,11 +1437,9 @@ define void @atomic16_nand_acq_rel(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acq_rel acquire, align 2, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 -1 acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -1538,8 +1475,7 @@ define void @atomic16_add_seq_cst(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] seq_cst seq_cst, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1558,8 +1494,7 @@ define void @atomic16_sub_seq_cst(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] seq_cst seq_cst, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1578,8 +1513,7 @@ define void @atomic16_and_seq_cst(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 0 seq_cst seq_cst, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1598,8 +1532,7 @@ define void @atomic16_or_seq_cst(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] seq_cst seq_cst, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1618,8 +1551,7 @@ define void @atomic16_xor_seq_cst(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[LOADED]] seq_cst seq_cst, align 2, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1638,11 +1570,9 @@ define void @atomic16_nand_seq_cst(i16* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] seq_cst seq_cst, align 2, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 -1 seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -1841,8 +1771,7 @@ define void @atomic32_add_monotonic(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] monotonic monotonic, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1861,8 +1790,7 @@ define void @atomic32_sub_monotonic(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] monotonic monotonic, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1881,8 +1809,7 @@ define void @atomic32_and_monotonic(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 0 monotonic monotonic, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1901,8 +1828,7 @@ define void @atomic32_or_monotonic(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] monotonic monotonic, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1921,8 +1847,7 @@ define void @atomic32_xor_monotonic(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] monotonic monotonic, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -1941,11 +1866,9 @@ define void @atomic32_nand_monotonic(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 -1 monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -1981,8 +1904,7 @@ define void @atomic32_add_acquire(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] acquire acquire, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2001,8 +1923,7 @@ define void @atomic32_sub_acquire(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] acquire acquire, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2021,8 +1942,7 @@ define void @atomic32_and_acquire(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 0 acquire acquire, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2041,8 +1961,7 @@ define void @atomic32_or_acquire(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] acquire acquire, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2061,8 +1980,7 @@ define void @atomic32_xor_acquire(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] acquire acquire, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2081,11 +1999,9 @@ define void @atomic32_nand_acquire(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acquire acquire, align 4, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 -1 acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -2121,8 +2037,7 @@ define void @atomic32_add_release(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] release monotonic, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2141,8 +2056,7 @@ define void @atomic32_sub_release(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] release monotonic, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2161,8 +2075,7 @@ define void @atomic32_and_release(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 0 release monotonic, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2181,8 +2094,7 @@ define void @atomic32_or_release(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] release monotonic, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2201,8 +2113,7 @@ define void @atomic32_xor_release(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] release monotonic, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2221,11 +2132,9 @@ define void @atomic32_nand_release(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 -1 release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -2261,8 +2170,7 @@ define void @atomic32_add_acq_rel(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] acq_rel acquire, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2281,8 +2189,7 @@ define void @atomic32_sub_acq_rel(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] acq_rel acquire, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2301,8 +2208,7 @@ define void @atomic32_and_acq_rel(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 0 acq_rel acquire, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2321,8 +2227,7 @@ define void @atomic32_or_acq_rel(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] acq_rel acquire, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2341,8 +2246,7 @@ define void @atomic32_xor_acq_rel(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] acq_rel acquire, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2361,11 +2265,9 @@ define void @atomic32_nand_acq_rel(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acq_rel acquire, align 4, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 -1 acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -2401,8 +2303,7 @@ define void @atomic32_add_seq_cst(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] seq_cst seq_cst, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2421,8 +2322,7 @@ define void @atomic32_sub_seq_cst(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] seq_cst seq_cst, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2441,8 +2341,7 @@ define void @atomic32_and_seq_cst(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 0 seq_cst seq_cst, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2461,8 +2360,7 @@ define void @atomic32_or_seq_cst(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] seq_cst seq_cst, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2481,8 +2379,7 @@ define void @atomic32_xor_seq_cst(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[LOADED]] seq_cst seq_cst, align 4, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2501,11 +2398,9 @@ define void @atomic32_nand_seq_cst(i32* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 -1 seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -2726,8 +2621,7 @@ define void @atomic64_add_monotonic(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] monotonic monotonic, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2746,8 +2640,7 @@ define void @atomic64_sub_monotonic(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] monotonic monotonic, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2766,8 +2659,7 @@ define void @atomic64_and_monotonic(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 0 monotonic monotonic, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2786,8 +2678,7 @@ define void @atomic64_or_monotonic(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] monotonic monotonic, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2806,8 +2697,7 @@ define void @atomic64_xor_monotonic(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] monotonic monotonic, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2826,11 +2716,9 @@ define void @atomic64_nand_monotonic(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] monotonic monotonic, align 8, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 -1 monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -2866,8 +2754,7 @@ define void @atomic64_add_acquire(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] acquire acquire, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2886,8 +2773,7 @@ define void @atomic64_sub_acquire(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] acquire acquire, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2906,8 +2792,7 @@ define void @atomic64_and_acquire(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 0 acquire acquire, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2926,8 +2811,7 @@ define void @atomic64_or_acquire(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] acquire acquire, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2946,8 +2830,7 @@ define void @atomic64_xor_acquire(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] acquire acquire, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -2966,11 +2849,9 @@ define void @atomic64_nand_acquire(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acquire acquire, align 8, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 -1 acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -3006,8 +2887,7 @@ define void @atomic64_add_release(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] release monotonic, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3026,8 +2906,7 @@ define void @atomic64_sub_release(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] release monotonic, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3046,8 +2925,7 @@ define void @atomic64_and_release(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 0 release monotonic, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3066,8 +2944,7 @@ define void @atomic64_or_release(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] release monotonic, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3086,8 +2963,7 @@ define void @atomic64_xor_release(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] release monotonic, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3106,11 +2982,9 @@ define void @atomic64_nand_release(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] release monotonic, align 8, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 -1 release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -3146,8 +3020,7 @@ define void @atomic64_add_acq_rel(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] acq_rel acquire, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3166,8 +3039,7 @@ define void @atomic64_sub_acq_rel(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] acq_rel acquire, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3186,8 +3058,7 @@ define void @atomic64_and_acq_rel(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 0 acq_rel acquire, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3206,8 +3077,7 @@ define void @atomic64_or_acq_rel(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] acq_rel acquire, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3226,8 +3096,7 @@ define void @atomic64_xor_acq_rel(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] acq_rel acquire, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3246,11 +3115,9 @@ define void @atomic64_nand_acq_rel(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acq_rel acquire, align 8, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 -1 acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -3286,8 +3153,7 @@ define void @atomic64_add_seq_cst(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] seq_cst seq_cst, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3306,8 +3172,7 @@ define void @atomic64_sub_seq_cst(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] seq_cst seq_cst, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3326,8 +3191,7 @@ define void @atomic64_and_seq_cst(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 0 seq_cst seq_cst, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3346,8 +3210,7 @@ define void @atomic64_or_seq_cst(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] seq_cst seq_cst, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3366,8 +3229,7 @@ define void @atomic64_xor_seq_cst(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[LOADED]] seq_cst seq_cst, align 8, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3386,11 +3248,9 @@ define void @atomic64_nand_seq_cst(i64* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 -1 seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -3644,8 +3504,7 @@ define void @atomic128_add_monotonic(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] monotonic monotonic, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3664,8 +3523,7 @@ define void @atomic128_sub_monotonic(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] monotonic monotonic, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3684,8 +3542,7 @@ define void @atomic128_and_monotonic(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 monotonic monotonic, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3704,8 +3561,7 @@ define void @atomic128_or_monotonic(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] monotonic monotonic, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3724,8 +3580,7 @@ define void @atomic128_xor_monotonic(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] monotonic monotonic, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3744,11 +3599,9 @@ define void @atomic128_nand_monotonic(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] monotonic monotonic, align 16, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 -1 monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -3784,8 +3637,7 @@ define void @atomic128_add_acquire(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] acquire acquire, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3804,8 +3656,7 @@ define void @atomic128_sub_acquire(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] acquire acquire, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3824,8 +3675,7 @@ define void @atomic128_and_acquire(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 acquire acquire, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3844,8 +3694,7 @@ define void @atomic128_or_acquire(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] acquire acquire, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3864,8 +3713,7 @@ define void @atomic128_xor_acquire(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] acquire acquire, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3884,11 +3732,9 @@ define void @atomic128_nand_acquire(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acquire acquire, align 16, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 -1 acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -3924,8 +3770,7 @@ define void @atomic128_add_release(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] release monotonic, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3944,8 +3789,7 @@ define void @atomic128_sub_release(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] release monotonic, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3964,8 +3808,7 @@ define void @atomic128_and_release(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 release monotonic, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -3984,8 +3827,7 @@ define void @atomic128_or_release(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] release monotonic, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -4004,8 +3846,7 @@ define void @atomic128_xor_release(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] release monotonic, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -4024,11 +3865,9 @@ define void @atomic128_nand_release(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] release monotonic, align 16, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 -1 release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -4064,8 +3903,7 @@ define void @atomic128_add_acq_rel(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] acq_rel acquire, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -4084,8 +3922,7 @@ define void @atomic128_sub_acq_rel(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] acq_rel acquire, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -4104,8 +3941,7 @@ define void @atomic128_and_acq_rel(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 acq_rel acquire, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -4124,8 +3960,7 @@ define void @atomic128_or_acq_rel(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] acq_rel acquire, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -4144,8 +3979,7 @@ define void @atomic128_xor_acq_rel(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] acq_rel acquire, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -4164,11 +3998,9 @@ define void @atomic128_nand_acq_rel(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acq_rel acquire, align 16, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 -1 acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void
@@ -4204,8 +4036,7 @@ define void @atomic128_add_seq_cst(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = add i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] seq_cst seq_cst, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -4224,8 +4055,7 @@ define void @atomic128_sub_seq_cst(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = sub i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] seq_cst seq_cst, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -4244,8 +4074,7 @@ define void @atomic128_and_seq_cst(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 seq_cst seq_cst, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -4264,8 +4093,7 @@ define void @atomic128_or_seq_cst(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = or i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] seq_cst seq_cst, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -4284,8 +4112,7 @@ define void @atomic128_xor_seq_cst(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[LOADED]] seq_cst seq_cst, align 16, !pcsections !0
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
@@ -4304,11 +4131,9 @@ define void @atomic128_nand_seq_cst(i128* %a) nounwind uwtable {
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
-; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[TMP1]], -1, !pcsections !0
-; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] seq_cst seq_cst, align 16, !pcsections !0
-; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1, !pcsections !0
-; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 -1 seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret void

diff  --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
index aef9c869be098..49e3c0691db97 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
@@ -39,20 +39,18 @@ define i16 @test_atomicrmw_xchg_i16_global_align4(i16 addrspace(1)* %ptr, i16 %v
 ; CHECK-LABEL: @test_atomicrmw_xchg_i16_global_align4(
 ; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = bitcast i16 addrspace(1)* [[PTR:%.*]] to i32 addrspace(1)*
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP1]], 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32 addrspace(1)* [[ALIGNEDADDR]], align 4
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[LOADED]], -65536
-; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP3]], [[VALOPERAND_SHIFTED]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP3]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP4]] seq_cst seq_cst, align 4
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; CHECK:       atomicrmw.end:
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], 0
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16
 ; CHECK-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw xchg i16 addrspace(1)* %ptr, i16 %value seq_cst, align 4
@@ -96,12 +94,11 @@ define i16 @test_atomicrmw_add_i16_global_align4(i16 addrspace(1)* %ptr, i16 %va
 ; CHECK-LABEL: @test_atomicrmw_add_i16_global_align4(
 ; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = bitcast i16 addrspace(1)* [[PTR:%.*]] to i32 addrspace(1)*
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP1]], 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32 addrspace(1)* [[ALIGNEDADDR]], align 4
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[NEW]], 65535
 ; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[LOADED]], -65536
 ; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP3]]
@@ -110,8 +107,7 @@ define i16 @test_atomicrmw_add_i16_global_align4(i16 addrspace(1)* %ptr, i16 %va
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; CHECK:       atomicrmw.end:
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], 0
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16
 ; CHECK-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw add i16 addrspace(1)* %ptr, i16 %value seq_cst, align 4
@@ -438,30 +434,27 @@ define i16 @test_cmpxchg_i16_global_align4(i16 addrspace(1)* %out, i16 %in, i16
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, i16 addrspace(1)* [[OUT:%.*]], i64 4
 ; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = bitcast i16 addrspace(1)* [[GEP]] to i32 addrspace(1)*
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[IN:%.*]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[OLD:%.*]] to i32
-; CHECK-NEXT:    [[TMP4:%.*]] = shl i32 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32 addrspace(1)* [[ALIGNEDADDR]], align 4
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP5]], -65536
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[OLD:%.*]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32 addrspace(1)* [[ALIGNEDADDR]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP3]], -65536
 ; CHECK-NEXT:    br label [[PARTWORD_CMPXCHG_LOOP:%.*]]
 ; CHECK:       partword.cmpxchg.loop:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP6]], [[TMP0:%.*]] ], [ [[TMP13:%.*]], [[PARTWORD_CMPXCHG_FAILURE:%.*]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = or i32 [[TMP7]], [[TMP4]]
-; CHECK-NEXT:    [[TMP10:%.*]] = cmpxchg i32 addrspace(1)* [[ALIGNEDADDR]], i32 [[TMP9]], i32 [[TMP8]] seq_cst seq_cst, align 4
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { i32, i1 } [[TMP10]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { i32, i1 } [[TMP10]], 1
-; CHECK-NEXT:    br i1 [[TMP12]], label [[PARTWORD_CMPXCHG_END:%.*]], label [[PARTWORD_CMPXCHG_FAILURE]]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[TMP11:%.*]], [[PARTWORD_CMPXCHG_FAILURE:%.*]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i32 [[TMP5]], [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = cmpxchg i32 addrspace(1)* [[ALIGNEDADDR]], i32 [[TMP7]], i32 [[TMP6]] seq_cst seq_cst, align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP8]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
+; CHECK-NEXT:    br i1 [[TMP10]], label [[PARTWORD_CMPXCHG_END:%.*]], label [[PARTWORD_CMPXCHG_FAILURE]]
 ; CHECK:       partword.cmpxchg.failure:
-; CHECK-NEXT:    [[TMP13]] = and i32 [[TMP11]], -65536
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP7]], [[TMP13]]
-; CHECK-NEXT:    br i1 [[TMP14]], label [[PARTWORD_CMPXCHG_LOOP]], label [[PARTWORD_CMPXCHG_END]]
+; CHECK-NEXT:    [[TMP11]] = and i32 [[TMP9]], -65536
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne i32 [[TMP5]], [[TMP11]]
+; CHECK-NEXT:    br i1 [[TMP12]], label [[PARTWORD_CMPXCHG_LOOP]], label [[PARTWORD_CMPXCHG_END]]
 ; CHECK:       partword.cmpxchg.end:
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP11]], 0
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
-; CHECK-NEXT:    [[TMP15:%.*]] = insertvalue { i16, i1 } undef, i16 [[EXTRACTED]], 0
-; CHECK-NEXT:    [[TMP16:%.*]] = insertvalue { i16, i1 } [[TMP15]], i1 [[TMP12]], 1
-; CHECK-NEXT:    [[EXTRACT:%.*]] = extractvalue { i16, i1 } [[TMP16]], 0
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP9]] to i16
+; CHECK-NEXT:    [[TMP13:%.*]] = insertvalue { i16, i1 } undef, i16 [[EXTRACTED]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = insertvalue { i16, i1 } [[TMP13]], i1 [[TMP10]], 1
+; CHECK-NEXT:    [[EXTRACT:%.*]] = extractvalue { i16, i1 } [[TMP14]], 0
 ; CHECK-NEXT:    ret i16 [[EXTRACT]]
 ;
   %gep = getelementptr i16, i16 addrspace(1)* %out, i64 4
@@ -547,10 +540,8 @@ define i16 @test_atomicrmw_xor_i16_local_align4(i16 addrspace(3)* %ptr, i16 %val
 ; CHECK-LABEL: @test_atomicrmw_xor_i16_local_align4(
 ; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = bitcast i16 addrspace(3)* [[PTR:%.*]] to i32 addrspace(3)*
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw xor i32 addrspace(3)* [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] seq_cst, align 4
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP2]], 0
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw xor i32 addrspace(3)* [[ALIGNEDADDR]], i32 [[TMP1]] seq_cst, align 4
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
 ; CHECK-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw xor i16 addrspace(3)* %ptr, i16 %value seq_cst, align 4

diff  --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8.ll
index 9c3e6ebf7c546..e7722013b8304 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8.ll
@@ -103,12 +103,11 @@ define i8 @test_atomicrmw_add_i8_global_align4(i8 addrspace(1)* %ptr, i8 %value)
 ; CHECK-LABEL: @test_atomicrmw_add_i8_global_align4(
 ; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = bitcast i8 addrspace(1)* [[PTR:%.*]] to i32 addrspace(1)*
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP1]], 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32 addrspace(1)* [[ALIGNEDADDR]], align 4
 ; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[NEW]], 255
 ; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[LOADED]], -256
 ; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP3]]
@@ -117,8 +116,7 @@ define i8 @test_atomicrmw_add_i8_global_align4(i8 addrspace(1)* %ptr, i8 %value)
 ; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
 ; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
 ; CHECK:       atomicrmw.end:
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], 0
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i8
 ; CHECK-NEXT:    ret i8 [[EXTRACTED]]
 ;
   %res = atomicrmw add i8 addrspace(1)* %ptr, i8 %value seq_cst, align 4

diff  --git a/llvm/test/Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll b/llvm/test/Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll
index 34026909d7642..7a2954ee213c7 100644
--- a/llvm/test/Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll
+++ b/llvm/test/Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll
@@ -14,8 +14,7 @@ define float @test_atomicrmw_fadd_f32(float* %ptr, float %value) {
 ; CHECK-NEXT:    [[STCX:%.*]] = call i32 @llvm.hexagon.S2.storew.locked(i32* [[TMP3]], i32 [[TMP4]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[STCX]], 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
-; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret float [[TMP2]]
 ;
@@ -36,8 +35,7 @@ define float @test_atomicrmw_fsub_f32(float* %ptr, float %value) {
 ; CHECK-NEXT:    [[STCX:%.*]] = call i32 @llvm.hexagon.S2.storew.locked(i32* [[TMP3]], i32 [[TMP4]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[STCX]], 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
-; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
 ; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret float [[TMP2]]
 ;

diff  --git a/llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll b/llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll
index 3e44f5ccbf37b..899f58ae56621 100644
--- a/llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll
+++ b/llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll
@@ -7,13 +7,11 @@
 define i1 @test_cmpxchg_seq_cst(i128* %addr, i128 %desire, i128 %new) {
 ; CHECK-LABEL: @test_cmpxchg_seq_cst(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMPVAL_SHIFTED:%.*]] = shl i128 [[DESIRE:%.*]], 0
-; CHECK-NEXT:    [[NEWVAL_SHIFTED:%.*]] = shl i128 [[NEW:%.*]], 0
-; CHECK-NEXT:    [[CMP_LO:%.*]] = trunc i128 [[CMPVAL_SHIFTED]] to i64
-; CHECK-NEXT:    [[TMP0:%.*]] = lshr i128 [[CMPVAL_SHIFTED]], 64
+; CHECK-NEXT:    [[CMP_LO:%.*]] = trunc i128 [[DESIRE:%.*]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i128 [[DESIRE]], 64
 ; CHECK-NEXT:    [[CMP_HI:%.*]] = trunc i128 [[TMP0]] to i64
-; CHECK-NEXT:    [[NEW_LO:%.*]] = trunc i128 [[NEWVAL_SHIFTED]] to i64
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i128 [[NEWVAL_SHIFTED]], 64
+; CHECK-NEXT:    [[NEW_LO:%.*]] = trunc i128 [[NEW:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i128 [[NEW]], 64
 ; CHECK-NEXT:    [[NEW_HI:%.*]] = trunc i128 [[TMP1]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i128* [[ADDR:%.*]] to i8*
 ; CHECK-NEXT:    call void @llvm.ppc.sync()
@@ -26,10 +24,9 @@ define i1 @test_cmpxchg_seq_cst(i128* %addr, i128 %desire, i128 %new) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl i128 [[HI64]], 64
 ; CHECK-NEXT:    [[VAL64:%.*]] = or i128 [[LO64]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertvalue { i128, i1 } undef, i128 [[VAL64]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = and i128 [[VAL64]], -1
-; CHECK-NEXT:    [[SUCCESS:%.*]] = icmp eq i128 [[CMPVAL_SHIFTED]], [[TMP6]]
-; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i128, i1 } [[TMP5]], i1 [[SUCCESS]], 1
-; CHECK-NEXT:    [[SUCC:%.*]] = extractvalue { i128, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = icmp eq i128 [[DESIRE]], [[VAL64]]
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i128, i1 } [[TMP5]], i1 [[SUCCESS]], 1
+; CHECK-NEXT:    [[SUCC:%.*]] = extractvalue { i128, i1 } [[TMP6]], 1
 ; CHECK-NEXT:    ret i1 [[SUCC]]
 ;
 ; PWR7-LABEL: @test_cmpxchg_seq_cst(

More information about the llvm-commits mailing list