[llvm] [SimplifyCFG] Emit SelectInst when folding branches to common dest with different PHI incoming values (PR #144434)

via llvm-commits llvm-commits at lists.llvm.org
Mon Jun 16 14:57:02 PDT 2025


https://github.com/HighW4y2H3ll updated https://github.com/llvm/llvm-project/pull/144434

>From 156c3b41f7faa8345355d2892d59c6dd58867037 Mon Sep 17 00:00:00 2001
From: h2h <h2h at meta.com>
Date: Mon, 16 Jun 2025 00:12:43 -0700
Subject: [PATCH 1/2] [SimplifyCFG] Emit SelectInst when folding branches to
 common dest with different PHI incoming values

---
 llvm/lib/Transforms/Utils/SimplifyCFG.cpp     |  82 ++++-
 llvm/test/CodeGen/AArch64/rm_redundant_cmp.ll |  56 ++--
 .../CodeGen/AArch64/tailmerging_in_mbp.ll     |   6 +-
 .../CodeGen/AArch64/typepromotion-cost.ll     |  49 ++-
 .../test/CodeGen/PowerPC/ppc-ctr-dead-code.ll |  32 +-
 llvm/test/CodeGen/X86/loop-search.ll          |  27 +-
 llvm/test/Transforms/LICM/hoist-phi.ll        |  28 +-
 llvm/test/Transforms/LICM/sinking.ll          |  56 ++--
 .../LoopIdiom/AArch64/byte-compare-index.ll   |  47 +--
 .../LoopIdiom/RISCV/byte-compare-index.ll     |  47 +--
 .../LoopStrengthReduce/AArch64/pr53625.ll     |  67 ++--
 .../AArch64/small-constant.ll                 |  40 +--
 .../LoopUnroll/runtime-loop-multiple-exits.ll | 290 ++++--------------
 llvm/test/Transforms/LoopUnroll/scevunroll.ll |  18 +-
 .../LoopUnroll/unroll-after-peel.ll           |   8 +-
 ...iting-with-phis-multiple-exiting-blocks.ll |  61 +---
 .../LoopVectorize/early_exit_legality.ll      |   4 +-
 .../single_early_exit_live_outs.ll            | 127 ++------
 .../LoopVectorize/unsupported_early_exit.ll   |  53 +++-
 .../inject-invariant-conditions.ll            |   4 +-
 .../2008-07-13-InfLoopMiscompile.ll           |   2 +-
 .../SimplifyCFG/X86/SpeculativeExec.ll        |   2 +-
 .../SimplifyCFG/X86/switch_to_lookup_table.ll |   5 +-
 .../fold-branch-to-common-dest-phis.ll        | 125 +++++---
 .../SimplifyCFG/fold-branch-to-common-dest.ll |  92 +++---
 .../Transforms/SimplifyCFG/hoist-dbgvalue.ll  |   2 +-
 .../SimplifyCFG/speculate-derefable-load.ll   |   2 +-
 .../SimplifyCFG/switch-on-const-select.ll     |   2 +-
 28 files changed, 606 insertions(+), 728 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 975ce3bef5176..3c4d14b169bfd 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -1170,6 +1170,9 @@ static void cloneInstructionsIntoPredecessorBlockAndUpdateSSAUses(
     // Note that we expect to be in a block-closed SSA form for this to work!
     for (Use &U : make_early_inc_range(BonusInst.uses())) {
       auto *UI = cast<Instruction>(U.getUser());
+      // Avoid dangling select instructions
+      if (!UI->getParent())
+        continue;
       auto *PN = dyn_cast<PHINode>(UI);
       if (!PN) {
         assert(UI->getParent() == BB && BonusInst.comesBefore(UI) &&
@@ -3965,7 +3968,8 @@ shouldFoldCondBranchesToCommonDestination(BranchInst *BI, BranchInst *PBI,
 static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
                                              DomTreeUpdater *DTU,
                                              MemorySSAUpdater *MSSAU,
-                                             const TargetTransformInfo *TTI) {
+                                             const TargetTransformInfo *TTI,
+                                             SmallDenseMap<PHINode*, SelectInst*, 8> &InsertNewPHIs) {
   BasicBlock *BB = BI->getParent();
   BasicBlock *PredBlock = PBI->getParent();
 
@@ -4052,6 +4056,28 @@ static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
 
   ValueToValueMapTy VMap; // maps original values to cloned values
   cloneInstructionsIntoPredecessorBlockAndUpdateSSAUses(BB, PredBlock, VMap);
+  if (!InsertNewPHIs.empty()) {
+    // Fixup PHINode in the commong successor
+    for (PHINode &PN : CommonSucc->phis()) {
+      auto It = InsertNewPHIs.find(&PN);
+      if (It != InsertNewPHIs.end() && It->first == &PN) {
+        Instruction *SI = It->second;
+        // Oprands might have been promoted to bonous inst
+        RemapInstruction(SI, VMap,
+                RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
+        // Insert SelectInst as the new PHINode incoming value
+        SI->insertBefore(PredBlock->getTerminator()->getIterator());
+        // Fix PHINode
+        PN.removeIncomingValue(PredBlock);
+        PN.addIncoming(SI, PredBlock);
+        // Remove map entry
+        InsertNewPHIs.erase(It);
+      }
+    }
+    // Cleanup dangling SelectInst
+    for (SelectInst *SI : InsertNewPHIs.values())
+      delete SI;
+  }
 
   Module *M = BB->getModule();
 
@@ -4111,15 +4137,48 @@ bool llvm::foldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
 
   // With which predecessors will we want to deal with?
   SmallVector<BasicBlock *, 8> Preds;
+  struct InsertPointTy {
+    InstructionCost Cost;
+    Value          *TValue; // True Value
+    Value          *FValue; // False Value
+    PHINode        *Phi;
+  };
+  SmallDenseMap<BranchInst*, SmallVector<InsertPointTy, 8>, 8> InsertPts;
   for (BasicBlock *PredBlock : predecessors(BB)) {
     BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator());
 
-    // Check that we have two conditional branches.  If there is a PHI node in
-    // the common successor, verify that the same value flows in from both
-    // blocks.
-    if (!PBI || PBI->isUnconditional() || !safeToMergeTerminators(BI, PBI))
+    // Check that we have two conditional branches.
+    if (!PBI || PBI->isUnconditional())
       continue;
 
+    // If there is a PHI node in the common successor, verify that the same value flows in from both
+    // blocks. Otherwise, check whether we can create a SelectInst to combine the incoming values
+    if (!safeToMergeTerminators(BI, PBI)) {
+      if (BI == PBI)
+        continue;
+      for (BasicBlock *Succ : BI->successors()) {
+        if (llvm::is_contained(PBI->successors(), Succ)) {
+          for (PHINode &Phi : Succ->phis()) {
+            Value *IV0 = Phi.getIncomingValueForBlock(BB);
+            Value *IV1 = Phi.getIncomingValueForBlock(PredBlock);
+            InstructionCost PCost;
+            if (TTI) {
+              PCost = TTI->getCmpSelInstrCost(Instruction::Select, Phi.getType(),
+                                     CmpInst::makeCmpResultType(Phi.getType()),
+                                     CmpInst::BAD_ICMP_PREDICATE, CostKind);
+            }
+            auto &IP = InsertPts[PBI];
+            if (PBI->getSuccessor(0) == BB)
+              IP.emplace_back(InsertPointTy{PCost, IV0, IV1, &Phi});
+            else
+              IP.emplace_back(InsertPointTy{PCost, IV1, IV0, &Phi});
+          }
+        }
+      }
+      if (InsertPts.empty())
+        continue;
+    }
+
     // Determine if the two branches share a common destination.
     BasicBlock *CommonSucc;
     Instruction::BinaryOps Opc;
@@ -4138,6 +4197,9 @@ bool llvm::foldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
                              !isa<CmpInst>(PBI->getCondition())))
         Cost += TTI->getArithmeticInstrCost(Instruction::Xor, Ty, CostKind);
 
+      for (auto const &InsertPoints : InsertPts.values())
+        for (auto &InsertInfo : InsertPoints)
+          Cost += InsertInfo.Cost;
       if (Cost > BranchFoldThreshold)
         continue;
     }
@@ -4203,7 +4265,15 @@ bool llvm::foldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
   // Ok, we have the budget. Perform the transformation.
   for (BasicBlock *PredBlock : Preds) {
     auto *PBI = cast<BranchInst>(PredBlock->getTerminator());
-    return performBranchToCommonDestFolding(BI, PBI, DTU, MSSAU, TTI);
+    SmallDenseMap<PHINode *, SelectInst *, 8> newPhis;
+    if (InsertPts.contains(PBI)) {
+      Value *PC = PBI->getCondition();
+      for (auto const InsertInfo: InsertPts[PBI]) {
+        SelectInst *newPhi = SelectInst::Create(PC, InsertInfo.TValue, InsertInfo.FValue);
+        newPhis.insert(std::make_pair(InsertInfo.Phi, newPhi));
+      }
+    }
+    return performBranchToCommonDestFolding(BI, PBI, DTU, MSSAU, TTI, newPhis);
   }
   return false;
 }
diff --git a/llvm/test/CodeGen/AArch64/rm_redundant_cmp.ll b/llvm/test/CodeGen/AArch64/rm_redundant_cmp.ll
index 9e3bb8ce8efc0..3240ef0a40ddc 100644
--- a/llvm/test/CodeGen/AArch64/rm_redundant_cmp.ll
+++ b/llvm/test/CodeGen/AArch64/rm_redundant_cmp.ll
@@ -49,16 +49,16 @@ define void @test_i16_2cmp_signed_2() {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    adrp x8, :got:cost_s_i8_i16
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:cost_s_i8_i16]
-; CHECK-NEXT:    ldrsh w9, [x8, #2]
-; CHECK-NEXT:    ldrsh w10, [x8, #4]
-; CHECK-NEXT:    cmp w9, w10
-; CHECK-NEXT:    b.gt .LBB1_2
-; CHECK-NEXT:  // %bb.1: // %if.else
-; CHECK-NEXT:    mov w9, w10
-; CHECK-NEXT:    b.ge .LBB1_3
-; CHECK-NEXT:  .LBB1_2: // %if.end8.sink.split
+; CHECK-NEXT:    ldrh w10, [x8, #2]
+; CHECK-NEXT:    ldrh w11, [x8, #4]
+; CHECK-NEXT:    sxth w9, w10
+; CHECK-NEXT:    cmp w9, w11, sxth
+; CHECK-NEXT:    csel w9, w10, w11, gt
+; CHECK-NEXT:    cmp w10, w11
+; CHECK-NEXT:    b.eq .LBB1_2
+; CHECK-NEXT:  // %bb.1: // %if.end8.sink.split
 ; CHECK-NEXT:    strh w9, [x8]
-; CHECK-NEXT:  .LBB1_3: // %if.end8
+; CHECK-NEXT:  .LBB1_2: // %if.end8
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i16, ptr getelementptr inbounds (%struct.s_signed_i16, ptr @cost_s_i8_i16, i64 0, i32 1), align 2
@@ -125,13 +125,11 @@ define void @test_i16_2cmp_unsigned_2() {
 ; CHECK-NEXT:    ldrh w9, [x8, #2]
 ; CHECK-NEXT:    ldrh w10, [x8, #4]
 ; CHECK-NEXT:    cmp w9, w10
-; CHECK-NEXT:    b.hi .LBB3_2
-; CHECK-NEXT:  // %bb.1: // %if.else
-; CHECK-NEXT:    mov w9, w10
-; CHECK-NEXT:    b.hs .LBB3_3
-; CHECK-NEXT:  .LBB3_2: // %if.end8.sink.split
+; CHECK-NEXT:    csel w9, w9, w10, hi
+; CHECK-NEXT:    b.eq .LBB3_2
+; CHECK-NEXT:  // %bb.1: // %if.end8.sink.split
 ; CHECK-NEXT:    strh w9, [x8]
-; CHECK-NEXT:  .LBB3_3: // %if.end8
+; CHECK-NEXT:  .LBB3_2: // %if.end8
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i16, ptr getelementptr inbounds (%struct.s_unsigned_i16, ptr @cost_u_i16, i64 0, i32 1), align 2
@@ -204,16 +202,16 @@ define void @test_i8_2cmp_signed_2() {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    adrp x8, :got:cost_s
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:cost_s]
-; CHECK-NEXT:    ldrsb w9, [x8, #1]
-; CHECK-NEXT:    ldrsb w10, [x8, #2]
-; CHECK-NEXT:    cmp w9, w10
-; CHECK-NEXT:    b.gt .LBB5_2
-; CHECK-NEXT:  // %bb.1: // %if.else
-; CHECK-NEXT:    mov w9, w10
-; CHECK-NEXT:    b.ge .LBB5_3
-; CHECK-NEXT:  .LBB5_2: // %if.end8.sink.split
+; CHECK-NEXT:    ldrb w10, [x8, #1]
+; CHECK-NEXT:    ldrb w11, [x8, #2]
+; CHECK-NEXT:    sxtb w9, w10
+; CHECK-NEXT:    cmp w9, w11, sxtb
+; CHECK-NEXT:    csel w9, w10, w11, gt
+; CHECK-NEXT:    cmp w10, w11
+; CHECK-NEXT:    b.eq .LBB5_2
+; CHECK-NEXT:  // %bb.1: // %if.end8.sink.split
 ; CHECK-NEXT:    strb w9, [x8]
-; CHECK-NEXT:  .LBB5_3: // %if.end8
+; CHECK-NEXT:  .LBB5_2: // %if.end8
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i8, ptr getelementptr inbounds (%struct.s_signed_i8, ptr @cost_s, i64 0, i32 1), align 2
@@ -280,13 +278,11 @@ define void @test_i8_2cmp_unsigned_2() {
 ; CHECK-NEXT:    ldrb w9, [x8, #1]
 ; CHECK-NEXT:    ldrb w10, [x8, #2]
 ; CHECK-NEXT:    cmp w9, w10
-; CHECK-NEXT:    b.hi .LBB7_2
-; CHECK-NEXT:  // %bb.1: // %if.else
-; CHECK-NEXT:    mov w9, w10
-; CHECK-NEXT:    b.hs .LBB7_3
-; CHECK-NEXT:  .LBB7_2: // %if.end8.sink.split
+; CHECK-NEXT:    csel w9, w9, w10, hi
+; CHECK-NEXT:    b.eq .LBB7_2
+; CHECK-NEXT:  // %bb.1: // %if.end8.sink.split
 ; CHECK-NEXT:    strb w9, [x8]
-; CHECK-NEXT:  .LBB7_3: // %if.end8
+; CHECK-NEXT:  .LBB7_2: // %if.end8
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i8, ptr getelementptr inbounds (%struct.s_unsigned_i8, ptr @cost_u_i8, i64 0, i32 1), align 2
diff --git a/llvm/test/CodeGen/AArch64/tailmerging_in_mbp.ll b/llvm/test/CodeGen/AArch64/tailmerging_in_mbp.ll
index 675380787af4d..75791f6589128 100644
--- a/llvm/test/CodeGen/AArch64/tailmerging_in_mbp.ll
+++ b/llvm/test/CodeGen/AArch64/tailmerging_in_mbp.ll
@@ -2,10 +2,10 @@
 
 ; CHECK-LABEL: test:
 ; CHECK-LABEL: %cond.false12.i
-; CHECK:         b.gt
-; CHECK-NEXT:  LBB0_8:
+; CHECK:         b.le
+; CHECK-LABEL:  LBB0_9:
 ; CHECK-NEXT:    mov	 x8, x9
-; CHECK-NEXT:  LBB0_9:
+; CHECK-NEXT:  LBB0_10:
 define i64 @test(i64 %n, ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f) {
 entry:
   %cmp28 = icmp sgt i64 %n, 1
diff --git a/llvm/test/CodeGen/AArch64/typepromotion-cost.ll b/llvm/test/CodeGen/AArch64/typepromotion-cost.ll
index 3aed4cb671c02..945588e5d2824 100644
--- a/llvm/test/CodeGen/AArch64/typepromotion-cost.ll
+++ b/llvm/test/CodeGen/AArch64/typepromotion-cost.ll
@@ -6,41 +6,28 @@
 define i32 @needless_promotion(ptr nocapture noundef readonly %S, i64 noundef %red_cost) {
 ; CHECK-O2-LABEL: needless_promotion:
 ; CHECK-O2:       // %bb.0: // %entry
-; CHECK-O2-NEXT:    ldrsh w8, [x0, #4]
-; CHECK-O2-NEXT:    tbnz w8, #31, .LBB0_3
-; CHECK-O2-NEXT:  // %bb.1: // %lor.rhs
-; CHECK-O2-NEXT:    cbz x1, .LBB0_5
-; CHECK-O2-NEXT:  // %bb.2:
-; CHECK-O2-NEXT:    mov w9, #2
-; CHECK-O2-NEXT:    b .LBB0_4
-; CHECK-O2-NEXT:  .LBB0_3:
-; CHECK-O2-NEXT:    mov w9, #1
-; CHECK-O2-NEXT:  .LBB0_4: // %lor.end.sink.split
-; CHECK-O2-NEXT:    cmp w8, w9
-; CHECK-O2-NEXT:    cset w0, eq
-; CHECK-O2-NEXT:    ret
-; CHECK-O2-NEXT:  .LBB0_5:
-; CHECK-O2-NEXT:    mov w0, wzr
+; CHECK-O2-NEXT:    ldrsh w9, [x0, #4]
+; CHECK-O2-NEXT:    mov w8, #1 // =0x1
+; CHECK-O2-NEXT:    cmp w9, #0
+; CHECK-O2-NEXT:    cinc w8, w8, ge
+; CHECK-O2-NEXT:    cmp w8, w9, uxth
+; CHECK-O2-NEXT:    cset w8, eq
+; CHECK-O2-NEXT:    cmp x1, #0
+; CHECK-O2-NEXT:    ccmn w9, #1, #4, eq
+; CHECK-O2-NEXT:    csel w0, wzr, w8, gt
 ; CHECK-O2-NEXT:    ret
 ;
 ; CHECK-O3-LABEL: needless_promotion:
 ; CHECK-O3:       // %bb.0: // %entry
-; CHECK-O3-NEXT:    ldrsh w8, [x0, #4]
-; CHECK-O3-NEXT:    tbnz w8, #31, .LBB0_3
-; CHECK-O3-NEXT:  // %bb.1: // %lor.rhs
-; CHECK-O3-NEXT:    cbz x1, .LBB0_4
-; CHECK-O3-NEXT:  // %bb.2:
-; CHECK-O3-NEXT:    mov w9, #2
-; CHECK-O3-NEXT:    cmp w8, w9
-; CHECK-O3-NEXT:    cset w0, eq
-; CHECK-O3-NEXT:    ret
-; CHECK-O3-NEXT:  .LBB0_3:
-; CHECK-O3-NEXT:    mov w9, #1
-; CHECK-O3-NEXT:    cmp w8, w9
-; CHECK-O3-NEXT:    cset w0, eq
-; CHECK-O3-NEXT:    ret
-; CHECK-O3-NEXT:  .LBB0_4:
-; CHECK-O3-NEXT:    mov w0, wzr
+; CHECK-O3-NEXT:    ldrsh w9, [x0, #4]
+; CHECK-O3-NEXT:    mov w8, #1 // =0x1
+; CHECK-O3-NEXT:    cmp w9, #0
+; CHECK-O3-NEXT:    cinc w8, w8, ge
+; CHECK-O3-NEXT:    cmp w8, w9, uxth
+; CHECK-O3-NEXT:    cset w8, eq
+; CHECK-O3-NEXT:    cmp x1, #0
+; CHECK-O3-NEXT:    ccmn w9, #1, #4, eq
+; CHECK-O3-NEXT:    csel w0, wzr, w8, gt
 ; CHECK-O3-NEXT:    ret
 entry:
   %ident = getelementptr inbounds %struct.S, ptr %S, i64 0, i32 1
diff --git a/llvm/test/CodeGen/PowerPC/ppc-ctr-dead-code.ll b/llvm/test/CodeGen/PowerPC/ppc-ctr-dead-code.ll
index bb8337d237f51..aceac8bb69b82 100644
--- a/llvm/test/CodeGen/PowerPC/ppc-ctr-dead-code.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc-ctr-dead-code.ll
@@ -8,25 +8,27 @@
 define signext i32 @limit_loop(i32 signext %iters, ptr nocapture readonly %vec, i32 signext %limit) local_unnamed_addr {
 ; CHECK-LABEL: limit_loop:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    mr 6, 3
-; CHECK-NEXT:    li 3, 0
-; CHECK-NEXT:    cmpwi 6, 0
-; CHECK-NEXT:    blelr 0
+; CHECK-NEXT:    cmpwi 3, 0
+; CHECK-NEXT:    ble 0, .LBB0_4
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
-; CHECK-NEXT:    mtctr 6
 ; CHECK-NEXT:    addi 4, 4, -4
-; CHECK-NEXT:    b .LBB0_3
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  .LBB0_2: # %for.cond
-; CHECK-NEXT:    #
-; CHECK-NEXT:    bdzlr
-; CHECK-NEXT:  .LBB0_3: # %for.body
+; CHECK-NEXT:    li 6, 1
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB0_2: # %for.body
 ; CHECK-NEXT:    #
-; CHECK-NEXT:    lwzu 6, 4(4)
-; CHECK-NEXT:    cmpw 6, 5
-; CHECK-NEXT:    blt 0, .LBB0_2
-; CHECK-NEXT:  # %bb.4:
+; CHECK-NEXT:    lwzu 7, 4(4)
+; CHECK-NEXT:    cmpd 1, 6, 3
+; CHECK-NEXT:    addi 6, 6, 1
+; CHECK-NEXT:    cmpw 7, 5
+; CHECK-NEXT:    crand 20, 0, 4
+; CHECK-NEXT:    bc 12, 20, .LBB0_2
+; CHECK-NEXT:  # %bb.3: # %cleanup.loopexit
 ; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    isellt 3, 0, 3
+; CHECK-NEXT:    clrldi 3, 3, 32
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB0_4:
+; CHECK-NEXT:    li 3, 0
 ; CHECK-NEXT:    blr
 entry:
   %cmp5 = icmp sgt i32 %iters, 0
diff --git a/llvm/test/CodeGen/X86/loop-search.ll b/llvm/test/CodeGen/X86/loop-search.ll
index 0d5f97d21fb3a..4fe5c48fcd26a 100644
--- a/llvm/test/CodeGen/X86/loop-search.ll
+++ b/llvm/test/CodeGen/X86/loop-search.ll
@@ -10,24 +10,29 @@ define zeroext i1 @search(i32 %needle, ptr nocapture readonly %haystack, i32 %co
 ; CHECK-NEXT:    testl %edx, %edx
 ; CHECK-NEXT:    jle LBB0_5
 ; CHECK-NEXT:  ## %bb.1: ## %for.body.preheader
-; CHECK-NEXT:    movslq %edx, %rax
-; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    movslq %edx, %rcx
+; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  LBB0_2: ## %for.body
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    cmpl %edi, (%rsi,%rcx,4)
-; CHECK-NEXT:    je LBB0_6
-; CHECK-NEXT:  ## %bb.3: ## %for.cond
+; CHECK-NEXT:    movl -4(%rsi,%rdx,4), %r8d
+; CHECK-NEXT:    cmpl %edi, %r8d
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    negb %al
+; CHECK-NEXT:    cmpl %edi, %r8d
+; CHECK-NEXT:    je LBB0_4
+; CHECK-NEXT:  ## %bb.3: ## %for.body
 ; CHECK-NEXT:    ## in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT:    incq %rcx
-; CHECK-NEXT:    cmpq %rax, %rcx
+; CHECK-NEXT:    cmpq %rcx, %rdx
+; CHECK-NEXT:    leaq 1(%rdx), %rdx
 ; CHECK-NEXT:    jl LBB0_2
-; CHECK-NEXT:  LBB0_5:
-; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:  LBB0_4: ## %cleanup
+; CHECK-NEXT:    andb $1, %al
 ; CHECK-NEXT:    ## kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    retq
-; CHECK-NEXT:  LBB0_6:
-; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:  LBB0_5:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    andb $1, %al
 ; CHECK-NEXT:    ## kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    retq
 entry:
diff --git a/llvm/test/Transforms/LICM/hoist-phi.ll b/llvm/test/Transforms/LICM/hoist-phi.ll
index bf999b98a1dac..3cedb14edfa30 100644
--- a/llvm/test/Transforms/LICM/hoist-phi.ll
+++ b/llvm/test/Transforms/LICM/hoist-phi.ll
@@ -629,16 +629,16 @@ define void @triangle_phi_loopexit(i32 %x, ptr %p) {
 ; CHECK-DISABLED-NEXT:  entry:
 ; CHECK-DISABLED-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], 1
 ; CHECK-DISABLED-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[X]], 0
+; CHECK-DISABLED-NEXT:    [[CMP1_NOT:%.*]] = xor i1 [[CMP1]], true
 ; CHECK-DISABLED-NEXT:    [[CMP2:%.*]] = icmp sgt i32 10, [[ADD]]
+; CHECK-DISABLED-NEXT:    [[TMP0:%.*]] = select i1 [[CMP1]], i32 [[ADD]], i32 [[X]]
+; CHECK-DISABLED-NEXT:    [[OR_COND:%.*]] = or i1 [[CMP1_NOT]], [[CMP2]]
+; CHECK-DISABLED-NEXT:    [[CMP3:%.*]] = icmp ne i32 [[TMP0]], 0
 ; CHECK-DISABLED-NEXT:    br label [[LOOP:%.*]]
 ; CHECK-DISABLED:       loop:
-; CHECK-DISABLED-NEXT:    br i1 [[CMP1]], label [[IF:%.*]], label [[THEN:%.*]]
-; CHECK-DISABLED:       if:
-; CHECK-DISABLED-NEXT:    br i1 [[CMP2]], label [[THEN]], label [[END:%.*]]
+; CHECK-DISABLED-NEXT:    br i1 [[OR_COND]], label [[THEN:%.*]], label [[END:%.*]]
 ; CHECK-DISABLED:       then:
-; CHECK-DISABLED-NEXT:    [[PHI:%.*]] = phi i32 [ [[ADD]], [[IF]] ], [ [[X]], [[LOOP]] ]
-; CHECK-DISABLED-NEXT:    store i32 [[PHI]], ptr [[P:%.*]], align 4
-; CHECK-DISABLED-NEXT:    [[CMP3:%.*]] = icmp ne i32 [[PHI]], 0
+; CHECK-DISABLED-NEXT:    store i32 [[TMP0]], ptr [[P:%.*]], align 4
 ; CHECK-DISABLED-NEXT:    br i1 [[CMP3]], label [[LOOP]], label [[END]]
 ; CHECK-DISABLED:       end:
 ; CHECK-DISABLED-NEXT:    ret void
@@ -647,20 +647,16 @@ define void @triangle_phi_loopexit(i32 %x, ptr %p) {
 ; CHECK-ENABLED-NEXT:  entry:
 ; CHECK-ENABLED-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], 1
 ; CHECK-ENABLED-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[X]], 0
+; CHECK-ENABLED-NEXT:    [[CMP1_NOT:%.*]] = xor i1 [[CMP1]], true
 ; CHECK-ENABLED-NEXT:    [[CMP2:%.*]] = icmp sgt i32 10, [[ADD]]
-; CHECK-ENABLED-NEXT:    br i1 [[CMP1]], label [[IF_LICM:%.*]], label [[THEN_LICM:%.*]]
-; CHECK-ENABLED:       if.licm:
-; CHECK-ENABLED-NEXT:    br label [[THEN_LICM]]
-; CHECK-ENABLED:       then.licm:
-; CHECK-ENABLED-NEXT:    [[PHI:%.*]] = phi i32 [ [[ADD]], [[IF_LICM]] ], [ [[X]], [[ENTRY:%.*]] ]
-; CHECK-ENABLED-NEXT:    [[CMP3:%.*]] = icmp ne i32 [[PHI]], 0
+; CHECK-ENABLED-NEXT:    [[TMP0:%.*]] = select i1 [[CMP1]], i32 [[ADD]], i32 [[X]]
+; CHECK-ENABLED-NEXT:    [[OR_COND:%.*]] = or i1 [[CMP1_NOT]], [[CMP2]]
+; CHECK-ENABLED-NEXT:    [[CMP3:%.*]] = icmp ne i32 [[TMP0]], 0
 ; CHECK-ENABLED-NEXT:    br label [[LOOP:%.*]]
 ; CHECK-ENABLED:       loop:
-; CHECK-ENABLED-NEXT:    br i1 [[CMP1]], label [[IF:%.*]], label [[THEN:%.*]]
-; CHECK-ENABLED:       if:
-; CHECK-ENABLED-NEXT:    br i1 [[CMP2]], label [[THEN]], label [[END:%.*]]
+; CHECK-ENABLED-NEXT:    br i1 [[OR_COND]], label [[THEN:%.*]], label [[END:%.*]]
 ; CHECK-ENABLED:       then:
-; CHECK-ENABLED-NEXT:    store i32 [[PHI]], ptr [[P:%.*]], align 4
+; CHECK-ENABLED-NEXT:    store i32 [[TMP0]], ptr [[P:%.*]], align 4
 ; CHECK-ENABLED-NEXT:    br i1 [[CMP3]], label [[LOOP]], label [[END]]
 ; CHECK-ENABLED:       end:
 ; CHECK-ENABLED-NEXT:    ret void
diff --git a/llvm/test/Transforms/LICM/sinking.ll b/llvm/test/Transforms/LICM/sinking.ll
index e7ac07b50625a..5d369afcd2578 100644
--- a/llvm/test/Transforms/LICM/sinking.ll
+++ b/llvm/test/Transforms/LICM/sinking.ll
@@ -80,7 +80,7 @@ define double @test2c(ptr %P) {
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br i1 true, label [[LOOP]], label [[OUT:%.*]]
 ; CHECK:       Out:
-; CHECK-NEXT:    [[A_LE:%.*]] = load double, ptr [[P:%.*]], align 8, !invariant.load !0
+; CHECK-NEXT:    [[A_LE:%.*]] = load double, ptr [[P:%.*]], align 8, !invariant.load [[META0:![0-9]+]]
 ; CHECK-NEXT:    ret double [[A_LE]]
 ;
   br label %Loop
@@ -535,23 +535,16 @@ define i32 @test14(i32 %N, i32 %N2, i1 %C) {
 ; CHECK-NEXT:  Entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       Loop:
-; CHECK-NEXT:    [[N_ADDR_0_PN:%.*]] = phi i32 [ [[DEC:%.*]], [[CONTLOOP:%.*]] ], [ [[N:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[N_ADDR_0_PN:%.*]] = phi i32 [ [[N:%.*]], [[ENTRY:%.*]] ], [ [[DEC:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[DEC]] = add i32 [[N_ADDR_0_PN]], -1
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[CONTLOOP]], label [[OUT12_SPLIT_LOOP_EXIT1:%.*]]
-; CHECK:       ContLoop:
 ; CHECK-NEXT:    [[TMP_1:%.*]] = icmp ne i32 [[N_ADDR_0_PN]], 1
-; CHECK-NEXT:    br i1 [[TMP_1]], label [[LOOP]], label [[OUT12_SPLIT_LOOP_EXIT:%.*]]
-; CHECK:       Out12.split.loop.exit:
-; CHECK-NEXT:    [[N_ADDR_0_PN_LCSSA4:%.*]] = phi i32 [ [[N_ADDR_0_PN]], [[CONTLOOP]] ]
-; CHECK-NEXT:    [[SINK_MUL_LE3:%.*]] = mul i32 [[N]], [[N_ADDR_0_PN_LCSSA4]]
-; CHECK-NEXT:    br label [[OUT12:%.*]]
-; CHECK:       Out12.split.loop.exit1:
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C:%.*]], i1 [[TMP_1]], i1 false
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[LOOP]], label [[OUT12:%.*]]
+; CHECK:       Out12:
 ; CHECK-NEXT:    [[N_ADDR_0_PN_LCSSA:%.*]] = phi i32 [ [[N_ADDR_0_PN]], [[LOOP]] ]
 ; CHECK-NEXT:    [[SINK_MUL_LE:%.*]] = mul i32 [[N]], [[N_ADDR_0_PN_LCSSA]]
 ; CHECK-NEXT:    [[SINK_SUB_LE:%.*]] = sub i32 [[SINK_MUL_LE]], [[N]]
-; CHECK-NEXT:    br label [[OUT12]]
-; CHECK:       Out12:
-; CHECK-NEXT:    [[TMP:%.*]] = phi i32 [ [[SINK_MUL_LE3]], [[OUT12_SPLIT_LOOP_EXIT]] ], [ [[SINK_SUB_LE]], [[OUT12_SPLIT_LOOP_EXIT1]] ]
+; CHECK-NEXT:    [[TMP:%.*]] = select i1 [[C]], i32 [[SINK_MUL_LE]], i32 [[SINK_SUB_LE]]
 ; CHECK-NEXT:    ret i32 [[TMP]]
 ;
 Entry:
@@ -581,24 +574,17 @@ define i32 @test15(i32 %N, i32 %N2, i1 %C) {
 ; CHECK-NEXT:  Entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       Loop:
-; CHECK-NEXT:    [[N_ADDR_0_PN:%.*]] = phi i32 [ [[DEC:%.*]], [[CONTLOOP:%.*]] ], [ [[N:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[N_ADDR_0_PN:%.*]] = phi i32 [ [[N:%.*]], [[ENTRY:%.*]] ], [ [[DEC:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[DEC]] = add i32 [[N_ADDR_0_PN]], -1
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[CONTLOOP]], label [[OUT12_SPLIT_LOOP_EXIT1:%.*]]
-; CHECK:       ContLoop:
 ; CHECK-NEXT:    [[TMP_1:%.*]] = icmp ne i32 [[N_ADDR_0_PN]], 1
-; CHECK-NEXT:    br i1 [[TMP_1]], label [[LOOP]], label [[OUT12_SPLIT_LOOP_EXIT:%.*]]
-; CHECK:       Out12.split.loop.exit:
-; CHECK-NEXT:    [[N_ADDR_0_PN_LCSSA5:%.*]] = phi i32 [ [[N_ADDR_0_PN]], [[CONTLOOP]] ]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C:%.*]], i1 [[TMP_1]], i1 false
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[LOOP]], label [[OUT12:%.*]]
+; CHECK:       Out12:
+; CHECK-NEXT:    [[N_ADDR_0_PN_LCSSA5:%.*]] = phi i32 [ [[N_ADDR_0_PN]], [[LOOP]] ]
 ; CHECK-NEXT:    [[SINK_MUL_LE4:%.*]] = mul i32 [[N]], [[N_ADDR_0_PN_LCSSA5]]
+; CHECK-NEXT:    [[SINK_SUB_LE:%.*]] = sub i32 [[SINK_MUL_LE4]], [[N]]
 ; CHECK-NEXT:    [[SINK_SUB2_LE:%.*]] = sub i32 [[SINK_MUL_LE4]], [[N2:%.*]]
-; CHECK-NEXT:    br label [[OUT12:%.*]]
-; CHECK:       Out12.split.loop.exit1:
-; CHECK-NEXT:    [[N_ADDR_0_PN_LCSSA:%.*]] = phi i32 [ [[N_ADDR_0_PN]], [[LOOP]] ]
-; CHECK-NEXT:    [[SINK_MUL_LE:%.*]] = mul i32 [[N]], [[N_ADDR_0_PN_LCSSA]]
-; CHECK-NEXT:    [[SINK_SUB_LE:%.*]] = sub i32 [[SINK_MUL_LE]], [[N]]
-; CHECK-NEXT:    br label [[OUT12]]
-; CHECK:       Out12:
-; CHECK-NEXT:    [[TMP:%.*]] = phi i32 [ [[SINK_SUB2_LE]], [[OUT12_SPLIT_LOOP_EXIT]] ], [ [[SINK_SUB_LE]], [[OUT12_SPLIT_LOOP_EXIT1]] ]
+; CHECK-NEXT:    [[TMP:%.*]] = select i1 [[C]], i32 [[SINK_SUB2_LE]], i32 [[SINK_SUB_LE]]
 ; CHECK-NEXT:    ret i32 [[TMP]]
 ;
 Entry:
@@ -630,9 +616,9 @@ define i32 @test16(i1 %c, ptr %P, ptr %P2, i64 %V) {
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[LOOP_PH]] ], [ [[NEXT:%.*]], [[CONTLOOP:%.*]] ]
 ; CHECK-NEXT:    [[L2:%.*]] = call i32 @getv()
 ; CHECK-NEXT:    switch i32 [[L2]], label [[CONTLOOP]] [
-; CHECK-NEXT:    i32 32, label [[OUT_SPLIT_LOOP_EXIT1:%.*]]
-; CHECK-NEXT:    i32 46, label [[OUT_SPLIT_LOOP_EXIT1]]
-; CHECK-NEXT:    i32 95, label [[OUT_SPLIT_LOOP_EXIT1]]
+; CHECK-NEXT:      i32 32, label [[OUT_SPLIT_LOOP_EXIT1:%.*]]
+; CHECK-NEXT:      i32 46, label [[OUT_SPLIT_LOOP_EXIT1]]
+; CHECK-NEXT:      i32 95, label [[OUT_SPLIT_LOOP_EXIT1]]
 ; CHECK-NEXT:    ]
 ; CHECK:       ContLoop:
 ; CHECK-NEXT:    [[NEXT]] = add nuw i64 [[IV]], 1
@@ -849,10 +835,10 @@ define void @test20(ptr %s, i1 %b, i32 %v1, i32 %v2) personality ptr @__CxxFrame
 ; CHECK-NEXT:    br i1 [[B:%.*]], label [[TRY_CONT:%.*]], label [[WHILE_BODY:%.*]]
 ; CHECK:       while.body:
 ; CHECK-NEXT:    invoke void @may_throw()
-; CHECK-NEXT:    to label [[WHILE_BODY2:%.*]] unwind label [[CATCH_DISPATCH:%.*]]
+; CHECK-NEXT:            to label [[WHILE_BODY2:%.*]] unwind label [[CATCH_DISPATCH:%.*]]
 ; CHECK:       while.body2:
 ; CHECK-NEXT:    invoke void @may_throw2()
-; CHECK-NEXT:    to label [[WHILE_COND]] unwind label [[CATCH_DISPATCH]]
+; CHECK-NEXT:            to label [[WHILE_COND]] unwind label [[CATCH_DISPATCH]]
 ; CHECK:       catch.dispatch:
 ; CHECK-NEXT:    [[DOTLCSSA1:%.*]] = phi i32 [ [[SINKABLE]], [[WHILE_BODY]] ], [ [[SINKABLE2]], [[WHILE_BODY2]] ]
 ; CHECK-NEXT:    [[CP:%.*]] = cleanuppad within none []
@@ -933,16 +919,16 @@ define void @test22(i1 %b, i32 %v1, i32 %v2) personality ptr @__CxxFrameHandler3
 ; CHECK-NEXT:    br i1 [[B:%.*]], label [[TRY_CONT:%.*]], label [[WHILE_BODY:%.*]]
 ; CHECK:       while.body:
 ; CHECK-NEXT:    invoke void @may_throw()
-; CHECK-NEXT:    to label [[WHILE_BODY2:%.*]] unwind label [[LPADBB:%.*]]
+; CHECK-NEXT:            to label [[WHILE_BODY2:%.*]] unwind label [[LPADBB:%.*]]
 ; CHECK:       while.body2:
 ; CHECK-NEXT:    [[V:%.*]] = call i32 @getv()
 ; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[V]], [[V2:%.*]]
 ; CHECK-NEXT:    invoke void @may_throw2()
-; CHECK-NEXT:    to label [[WHILE_COND]] unwind label [[LPADBB]]
+; CHECK-NEXT:            to label [[WHILE_COND]] unwind label [[LPADBB]]
 ; CHECK:       lpadBB:
 ; CHECK-NEXT:    [[DOTLCSSA1:%.*]] = phi i32 [ 0, [[WHILE_BODY]] ], [ [[MUL]], [[WHILE_BODY2]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = landingpad { ptr, i32 }
-; CHECK-NEXT:    catch ptr null
+; CHECK-NEXT:            catch ptr null
 ; CHECK-NEXT:    br label [[LPADBBSUCC1:%.*]]
 ; CHECK:       lpadBBSucc1:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll b/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll
index 39037761c81bb..00cea91a67709 100644
--- a/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll
+++ b/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll
@@ -160,20 +160,21 @@ define i32 @compare_bytes_simple(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
 ; LOOP-DEL:       mismatch_loop_pre:
 ; LOOP-DEL-NEXT:    br label [[MISMATCH_LOOP:%.*]]
 ; LOOP-DEL:       mismatch_loop:
-; LOOP-DEL-NEXT:    [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP43:%.*]], [[MISMATCH_LOOP_INC:%.*]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP43:%.*]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    [[TMP37:%.*]] = zext i32 [[MISMATCH_INDEX]] to i64
 ; LOOP-DEL-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP37]]
 ; LOOP-DEL-NEXT:    [[TMP39:%.*]] = load i8, ptr [[TMP38]], align 1
 ; LOOP-DEL-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP37]]
 ; LOOP-DEL-NEXT:    [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
 ; LOOP-DEL-NEXT:    [[TMP42:%.*]] = icmp eq i8 [[TMP39]], [[TMP41]]
-; LOOP-DEL-NEXT:    br i1 [[TMP42]], label [[MISMATCH_LOOP_INC]], label [[WHILE_END]]
-; LOOP-DEL:       mismatch_loop_inc:
+; LOOP-DEL-NEXT:    [[DOTNOT:%.*]] = xor i1 [[TMP42]], true
 ; LOOP-DEL-NEXT:    [[TMP43]] = add i32 [[MISMATCH_INDEX]], 1
 ; LOOP-DEL-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
-; LOOP-DEL-NEXT:    br i1 [[TMP44]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
+; LOOP-DEL-NEXT:    [[TMP45:%.*]] = select i1 [[TMP42]], i32 [[N]], i32 [[MISMATCH_INDEX]]
+; LOOP-DEL-NEXT:    [[OR_COND:%.*]] = or i1 [[DOTNOT]], [[TMP44]]
+; LOOP-DEL-NEXT:    br i1 [[OR_COND]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
 ; LOOP-DEL:       while.end:
-; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ], [ [[TMP45]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    [[RES:%.*]] = add i32 [[MISMATCH_RESULT]], [[EXTRA]]
 ; LOOP-DEL-NEXT:    ret i32 [[RES]]
 ;
@@ -381,20 +382,21 @@ define i32 @compare_bytes_signed_wrap(ptr %a, ptr %b, i32 %len, i32 %n) {
 ; LOOP-DEL:       mismatch_loop_pre:
 ; LOOP-DEL-NEXT:    br label [[MISMATCH_LOOP:%.*]]
 ; LOOP-DEL:       mismatch_loop:
-; LOOP-DEL-NEXT:    [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP43:%.*]], [[MISMATCH_LOOP_INC:%.*]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP43:%.*]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    [[TMP37:%.*]] = zext i32 [[MISMATCH_INDEX]] to i64
 ; LOOP-DEL-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP37]]
 ; LOOP-DEL-NEXT:    [[TMP39:%.*]] = load i8, ptr [[TMP38]], align 1
 ; LOOP-DEL-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP37]]
 ; LOOP-DEL-NEXT:    [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
 ; LOOP-DEL-NEXT:    [[TMP42:%.*]] = icmp eq i8 [[TMP39]], [[TMP41]]
-; LOOP-DEL-NEXT:    br i1 [[TMP42]], label [[MISMATCH_LOOP_INC]], label [[WHILE_END]]
-; LOOP-DEL:       mismatch_loop_inc:
+; LOOP-DEL-NEXT:    [[DOTNOT:%.*]] = xor i1 [[TMP42]], true
 ; LOOP-DEL-NEXT:    [[TMP43]] = add nsw i32 [[MISMATCH_INDEX]], 1
 ; LOOP-DEL-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
-; LOOP-DEL-NEXT:    br i1 [[TMP44]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
+; LOOP-DEL-NEXT:    [[TMP45:%.*]] = select i1 [[TMP42]], i32 [[N]], i32 [[MISMATCH_INDEX]]
+; LOOP-DEL-NEXT:    [[OR_COND:%.*]] = select i1 [[DOTNOT]], i1 true, i1 [[TMP44]]
+; LOOP-DEL-NEXT:    br i1 [[OR_COND]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
 ; LOOP-DEL:       while.end:
-; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ], [ [[TMP45]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    ret i32 [[MISMATCH_RESULT]]
 ;
 ; NO-TRANSFORM-LABEL: define i32 @compare_bytes_signed_wrap(
@@ -608,20 +610,21 @@ define i32 @compare_bytes_simple_end_ne_found(ptr %a, ptr %b, ptr %c, ptr %d, i3
 ; LOOP-DEL:       mismatch_loop_pre:
 ; LOOP-DEL-NEXT:    br label [[MISMATCH_LOOP:%.*]]
 ; LOOP-DEL:       mismatch_loop:
-; LOOP-DEL-NEXT:    [[MISMATCH_INDEX3:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP43:%.*]], [[MISMATCH_LOOP_INC:%.*]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_INDEX3:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP43:%.*]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    [[TMP37:%.*]] = zext i32 [[MISMATCH_INDEX3]] to i64
 ; LOOP-DEL-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP37]]
 ; LOOP-DEL-NEXT:    [[TMP39:%.*]] = load i8, ptr [[TMP38]], align 1
 ; LOOP-DEL-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP37]]
 ; LOOP-DEL-NEXT:    [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
 ; LOOP-DEL-NEXT:    [[TMP42:%.*]] = icmp eq i8 [[TMP39]], [[TMP41]]
-; LOOP-DEL-NEXT:    br i1 [[TMP42]], label [[MISMATCH_LOOP_INC]], label [[BYTE_COMPARE]]
-; LOOP-DEL:       mismatch_loop_inc:
+; LOOP-DEL-NEXT:    [[DOTNOT:%.*]] = xor i1 [[TMP42]], true
 ; LOOP-DEL-NEXT:    [[TMP43]] = add i32 [[MISMATCH_INDEX3]], 1
 ; LOOP-DEL-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
-; LOOP-DEL-NEXT:    br i1 [[TMP44]], label [[BYTE_COMPARE]], label [[MISMATCH_LOOP]]
+; LOOP-DEL-NEXT:    [[TMP46:%.*]] = select i1 [[TMP42]], i32 [[N]], i32 [[MISMATCH_INDEX3]]
+; LOOP-DEL-NEXT:    [[OR_COND:%.*]] = or i1 [[DOTNOT]], [[TMP44]]
+; LOOP-DEL-NEXT:    br i1 [[OR_COND]], label [[BYTE_COMPARE]], label [[MISMATCH_LOOP]]
 ; LOOP-DEL:       byte.compare:
-; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX3]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ], [ [[TMP46]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    [[TMP45:%.*]] = icmp eq i32 [[MISMATCH_RESULT]], [[N]]
 ; LOOP-DEL-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP45]], i32 [[N]], i32 [[MISMATCH_RESULT]]
 ; LOOP-DEL-NEXT:    [[SPEC_SELECT4:%.*]] = select i1 [[TMP45]], ptr [[D]], ptr [[C]]
@@ -860,20 +863,21 @@ define i32 @compare_bytes_extra_cmp(ptr %a, ptr %b, i32 %len, i32 %n, i32 %x) {
 ; LOOP-DEL:       mismatch_loop_pre:
 ; LOOP-DEL-NEXT:    br label [[MISMATCH_LOOP:%.*]]
 ; LOOP-DEL:       mismatch_loop:
-; LOOP-DEL-NEXT:    [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP43:%.*]], [[MISMATCH_LOOP_INC:%.*]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP43:%.*]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    [[TMP37:%.*]] = zext i32 [[MISMATCH_INDEX]] to i64
 ; LOOP-DEL-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP37]]
 ; LOOP-DEL-NEXT:    [[TMP39:%.*]] = load i8, ptr [[TMP38]], align 1
 ; LOOP-DEL-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP37]]
 ; LOOP-DEL-NEXT:    [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
 ; LOOP-DEL-NEXT:    [[TMP42:%.*]] = icmp eq i8 [[TMP39]], [[TMP41]]
-; LOOP-DEL-NEXT:    br i1 [[TMP42]], label [[MISMATCH_LOOP_INC]], label [[WHILE_END]]
-; LOOP-DEL:       mismatch_loop_inc:
+; LOOP-DEL-NEXT:    [[DOTNOT:%.*]] = xor i1 [[TMP42]], true
 ; LOOP-DEL-NEXT:    [[TMP43]] = add i32 [[MISMATCH_INDEX]], 1
 ; LOOP-DEL-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
-; LOOP-DEL-NEXT:    br i1 [[TMP44]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
+; LOOP-DEL-NEXT:    [[TMP45:%.*]] = select i1 [[TMP42]], i32 [[N]], i32 [[MISMATCH_INDEX]]
+; LOOP-DEL-NEXT:    [[OR_COND:%.*]] = or i1 [[DOTNOT]], [[TMP44]]
+; LOOP-DEL-NEXT:    br i1 [[OR_COND]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
 ; LOOP-DEL:       while.end:
-; LOOP-DEL-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ]
+; LOOP-DEL-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ [[N]], [[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], [[MISMATCH_VEC_LOOP_FOUND]] ], [ [[TMP45]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    ret i32 [[INC_LCSSA]]
 ;
 ; NO-TRANSFORM-LABEL: define i32 @compare_bytes_extra_cmp(
@@ -1040,7 +1044,8 @@ define void @compare_bytes_cleanup_block(ptr %src1, ptr %src2) {
 ; LOOP-DEL-NEXT:    [[TMP2:%.*]] = load i8, ptr [[TMP1]], align 1
 ; LOOP-DEL-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[SRC2]], i64 [[TMP0]]
 ; LOOP-DEL-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
-; LOOP-DEL-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP2]], [[TMP4]]
+; LOOP-DEL-NEXT:    [[TMP8:%.*]] = icmp eq i8 [[TMP2]], [[TMP4]]
+; LOOP-DEL-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP8]], true
 ; LOOP-DEL-NEXT:    [[TMP6]] = add i32 [[MISMATCH_INDEX]], 1
 ; LOOP-DEL-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
 ; LOOP-DEL-NEXT:    [[OR_COND:%.*]] = or i1 [[TMP5]], [[TMP7]]
diff --git a/llvm/test/Transforms/LoopIdiom/RISCV/byte-compare-index.ll b/llvm/test/Transforms/LoopIdiom/RISCV/byte-compare-index.ll
index afc28cfda45aa..7060aded5012b 100644
--- a/llvm/test/Transforms/LoopIdiom/RISCV/byte-compare-index.ll
+++ b/llvm/test/Transforms/LoopIdiom/RISCV/byte-compare-index.ll
@@ -237,20 +237,21 @@ define i32 @compare_bytes_simple(ptr %a, ptr %b, i32 %len, i32 %n) {
 ; LOOP-DEL:       mismatch_loop_pre:
 ; LOOP-DEL-NEXT:    br label [[MISMATCH_LOOP:%.*]]
 ; LOOP-DEL:       mismatch_loop:
-; LOOP-DEL-NEXT:    [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP35:%.*]], [[MISMATCH_LOOP_INC:%.*]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP35:%.*]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    [[TMP29:%.*]] = zext i32 [[MISMATCH_INDEX]] to i64
 ; LOOP-DEL-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP29]]
 ; LOOP-DEL-NEXT:    [[TMP31:%.*]] = load i8, ptr [[TMP30]], align 1
 ; LOOP-DEL-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP29]]
 ; LOOP-DEL-NEXT:    [[TMP33:%.*]] = load i8, ptr [[TMP32]], align 1
 ; LOOP-DEL-NEXT:    [[TMP34:%.*]] = icmp eq i8 [[TMP31]], [[TMP33]]
-; LOOP-DEL-NEXT:    br i1 [[TMP34]], label [[MISMATCH_LOOP_INC]], label [[WHILE_END]]
-; LOOP-DEL:       mismatch_loop_inc:
+; LOOP-DEL-NEXT:    [[DOTNOT:%.*]] = xor i1 [[TMP34]], true
 ; LOOP-DEL-NEXT:    [[TMP35]] = add i32 [[MISMATCH_INDEX]], 1
 ; LOOP-DEL-NEXT:    [[TMP36:%.*]] = icmp eq i32 [[TMP35]], [[N]]
-; LOOP-DEL-NEXT:    br i1 [[TMP36]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
+; LOOP-DEL-NEXT:    [[TMP38:%.*]] = select i1 [[TMP34]], i32 [[N]], i32 [[MISMATCH_INDEX]]
+; LOOP-DEL-NEXT:    [[OR_COND:%.*]] = or i1 [[DOTNOT]], [[TMP36]]
+; LOOP-DEL-NEXT:    br i1 [[OR_COND]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
 ; LOOP-DEL:       while.end:
-; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VECTOR_LOOP_INC]] ], [ [[TMP28]], [[MISMATCH_VECTOR_LOOP_FOUND]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_VECTOR_LOOP_INC]] ], [ [[TMP28]], [[MISMATCH_VECTOR_LOOP_FOUND]] ], [ [[TMP38]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    ret i32 [[MISMATCH_RESULT]]
 ;
 ; MASKED-LABEL: define i32 @compare_bytes_simple(
@@ -604,20 +605,21 @@ define i32 @compare_bytes_signed_wrap(ptr %a, ptr %b, i32 %len, i32 %n) {
 ; LOOP-DEL:       mismatch_loop_pre:
 ; LOOP-DEL-NEXT:    br label [[MISMATCH_LOOP:%.*]]
 ; LOOP-DEL:       mismatch_loop:
-; LOOP-DEL-NEXT:    [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP35:%.*]], [[MISMATCH_LOOP_INC:%.*]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP35:%.*]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    [[TMP29:%.*]] = zext i32 [[MISMATCH_INDEX]] to i64
 ; LOOP-DEL-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP29]]
 ; LOOP-DEL-NEXT:    [[TMP31:%.*]] = load i8, ptr [[TMP30]], align 1
 ; LOOP-DEL-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP29]]
 ; LOOP-DEL-NEXT:    [[TMP33:%.*]] = load i8, ptr [[TMP32]], align 1
 ; LOOP-DEL-NEXT:    [[TMP34:%.*]] = icmp eq i8 [[TMP31]], [[TMP33]]
-; LOOP-DEL-NEXT:    br i1 [[TMP34]], label [[MISMATCH_LOOP_INC]], label [[WHILE_END]]
-; LOOP-DEL:       mismatch_loop_inc:
+; LOOP-DEL-NEXT:    [[DOTNOT:%.*]] = xor i1 [[TMP34]], true
 ; LOOP-DEL-NEXT:    [[TMP35]] = add nsw i32 [[MISMATCH_INDEX]], 1
 ; LOOP-DEL-NEXT:    [[TMP36:%.*]] = icmp eq i32 [[TMP35]], [[N]]
-; LOOP-DEL-NEXT:    br i1 [[TMP36]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
+; LOOP-DEL-NEXT:    [[TMP38:%.*]] = select i1 [[TMP34]], i32 [[N]], i32 [[MISMATCH_INDEX]]
+; LOOP-DEL-NEXT:    [[OR_COND:%.*]] = select i1 [[DOTNOT]], i1 true, i1 [[TMP36]]
+; LOOP-DEL-NEXT:    br i1 [[OR_COND]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
 ; LOOP-DEL:       while.end:
-; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VECTOR_LOOP_INC]] ], [ [[TMP28]], [[MISMATCH_VECTOR_LOOP_FOUND]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_VECTOR_LOOP_INC]] ], [ [[TMP28]], [[MISMATCH_VECTOR_LOOP_FOUND]] ], [ [[TMP38]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    ret i32 [[MISMATCH_RESULT]]
 ;
 ; MASKED-LABEL: define i32 @compare_bytes_signed_wrap(
@@ -1014,20 +1016,21 @@ define i32 @compare_bytes_simple_end_ne_found(ptr %a, ptr %b, ptr %c, ptr %d, i3
 ; LOOP-DEL:       mismatch_loop_pre:
 ; LOOP-DEL-NEXT:    br label [[MISMATCH_LOOP:%.*]]
 ; LOOP-DEL:       mismatch_loop:
-; LOOP-DEL-NEXT:    [[MISMATCH_INDEX3:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP35:%.*]], [[MISMATCH_LOOP_INC:%.*]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_INDEX3:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP35:%.*]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    [[TMP29:%.*]] = zext i32 [[MISMATCH_INDEX3]] to i64
 ; LOOP-DEL-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP29]]
 ; LOOP-DEL-NEXT:    [[TMP31:%.*]] = load i8, ptr [[TMP30]], align 1
 ; LOOP-DEL-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP29]]
 ; LOOP-DEL-NEXT:    [[TMP33:%.*]] = load i8, ptr [[TMP32]], align 1
 ; LOOP-DEL-NEXT:    [[TMP34:%.*]] = icmp eq i8 [[TMP31]], [[TMP33]]
-; LOOP-DEL-NEXT:    br i1 [[TMP34]], label [[MISMATCH_LOOP_INC]], label [[BYTE_COMPARE]]
-; LOOP-DEL:       mismatch_loop_inc:
+; LOOP-DEL-NEXT:    [[DOTNOT:%.*]] = xor i1 [[TMP34]], true
 ; LOOP-DEL-NEXT:    [[TMP35]] = add i32 [[MISMATCH_INDEX3]], 1
 ; LOOP-DEL-NEXT:    [[TMP36:%.*]] = icmp eq i32 [[TMP35]], [[N]]
-; LOOP-DEL-NEXT:    br i1 [[TMP36]], label [[BYTE_COMPARE]], label [[MISMATCH_LOOP]]
+; LOOP-DEL-NEXT:    [[TMP38:%.*]] = select i1 [[TMP34]], i32 [[N]], i32 [[MISMATCH_INDEX3]]
+; LOOP-DEL-NEXT:    [[OR_COND:%.*]] = or i1 [[DOTNOT]], [[TMP36]]
+; LOOP-DEL-NEXT:    br i1 [[OR_COND]], label [[BYTE_COMPARE]], label [[MISMATCH_LOOP]]
 ; LOOP-DEL:       byte.compare:
-; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX3]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VECTOR_LOOP_INC]] ], [ [[TMP28]], [[MISMATCH_VECTOR_LOOP_FOUND]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], [[MISMATCH_VECTOR_LOOP_INC]] ], [ [[TMP28]], [[MISMATCH_VECTOR_LOOP_FOUND]] ], [ [[TMP38]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    [[TMP37:%.*]] = icmp eq i32 [[MISMATCH_RESULT]], [[N]]
 ; LOOP-DEL-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP37]], i32 [[N]], i32 [[MISMATCH_RESULT]]
 ; LOOP-DEL-NEXT:    [[SPEC_SELECT4:%.*]] = select i1 [[TMP37]], ptr [[D]], ptr [[C]]
@@ -1455,20 +1458,21 @@ define i32 @compare_bytes_extra_cmp(ptr %a, ptr %b, i32 %len, i32 %n, i32 %x) {
 ; LOOP-DEL:       mismatch_loop_pre:
 ; LOOP-DEL-NEXT:    br label [[MISMATCH_LOOP:%.*]]
 ; LOOP-DEL:       mismatch_loop:
-; LOOP-DEL-NEXT:    [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP35:%.*]], [[MISMATCH_LOOP_INC:%.*]] ]
+; LOOP-DEL-NEXT:    [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], [[MISMATCH_LOOP_PRE]] ], [ [[TMP35:%.*]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    [[TMP29:%.*]] = zext i32 [[MISMATCH_INDEX]] to i64
 ; LOOP-DEL-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP29]]
 ; LOOP-DEL-NEXT:    [[TMP31:%.*]] = load i8, ptr [[TMP30]], align 1
 ; LOOP-DEL-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP29]]
 ; LOOP-DEL-NEXT:    [[TMP33:%.*]] = load i8, ptr [[TMP32]], align 1
 ; LOOP-DEL-NEXT:    [[TMP34:%.*]] = icmp eq i8 [[TMP31]], [[TMP33]]
-; LOOP-DEL-NEXT:    br i1 [[TMP34]], label [[MISMATCH_LOOP_INC]], label [[WHILE_END]]
-; LOOP-DEL:       mismatch_loop_inc:
+; LOOP-DEL-NEXT:    [[DOTNOT:%.*]] = xor i1 [[TMP34]], true
 ; LOOP-DEL-NEXT:    [[TMP35]] = add i32 [[MISMATCH_INDEX]], 1
 ; LOOP-DEL-NEXT:    [[TMP36:%.*]] = icmp eq i32 [[TMP35]], [[N]]
-; LOOP-DEL-NEXT:    br i1 [[TMP36]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
+; LOOP-DEL-NEXT:    [[TMP38:%.*]] = select i1 [[TMP34]], i32 [[N]], i32 [[MISMATCH_INDEX]]
+; LOOP-DEL-NEXT:    [[OR_COND:%.*]] = or i1 [[DOTNOT]], [[TMP36]]
+; LOOP-DEL-NEXT:    br i1 [[OR_COND]], label [[WHILE_END]], label [[MISMATCH_LOOP]]
 ; LOOP-DEL:       while.end:
-; LOOP-DEL-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ [[N]], [[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], [[MISMATCH_LOOP]] ], [ [[N]], [[MISMATCH_VECTOR_LOOP_INC]] ], [ [[TMP28]], [[MISMATCH_VECTOR_LOOP_FOUND]] ]
+; LOOP-DEL-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ [[N]], [[MISMATCH_VECTOR_LOOP_INC]] ], [ [[TMP28]], [[MISMATCH_VECTOR_LOOP_FOUND]] ], [ [[TMP38]], [[MISMATCH_LOOP]] ]
 ; LOOP-DEL-NEXT:    ret i32 [[INC_LCSSA]]
 ;
 ; MASKED-LABEL: define i32 @compare_bytes_extra_cmp(
@@ -1812,7 +1816,8 @@ define void @compare_bytes_cleanup_block(ptr %src1, ptr %src2) {
 ; LOOP-DEL-NEXT:    [[TMP2:%.*]] = load i8, ptr [[TMP1]], align 1
 ; LOOP-DEL-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[SRC2]], i64 [[TMP0]]
 ; LOOP-DEL-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
-; LOOP-DEL-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP2]], [[TMP4]]
+; LOOP-DEL-NEXT:    [[TMP8:%.*]] = icmp eq i8 [[TMP2]], [[TMP4]]
+; LOOP-DEL-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP8]], true
 ; LOOP-DEL-NEXT:    [[TMP6]] = add i32 [[MISMATCH_INDEX]], 1
 ; LOOP-DEL-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
 ; LOOP-DEL-NEXT:    [[OR_COND:%.*]] = or i1 [[TMP5]], [[TMP7]]
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/pr53625.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/pr53625.ll
index 3c1094f2ee31d..5787f6c041b63 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/AArch64/pr53625.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/pr53625.ll
@@ -8,22 +8,25 @@ define i32 @test(i32 %c, ptr %a, ptr %b) {
 ; CHECK-LABEL: test:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    cmp w0, #1
-; CHECK-NEXT:    b.lt .LBB0_4
+; CHECK-NEXT:    b.lt .LBB0_5
 ; CHECK-NEXT:  // %bb.1: // %for.body.preheader
 ; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    sub x8, x8, #1
 ; CHECK-NEXT:  .LBB0_2: // %for.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr w9, [x1], #4
-; CHECK-NEXT:    cbnz w9, .LBB0_5
-; CHECK-NEXT:  // %bb.3: // %for.cond
+; CHECK-NEXT:    ldr w10, [x1], #4
+; CHECK-NEXT:    cmp w10, #0
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    cbnz w10, .LBB0_4
+; CHECK-NEXT:  // %bb.3: // %for.body
 ; CHECK-NEXT:    // in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT:    subs x8, x8, #1
-; CHECK-NEXT:    b.ne .LBB0_2
-; CHECK-NEXT:  .LBB0_4:
-; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    mov x9, x8
+; CHECK-NEXT:    sub x8, x8, #1
+; CHECK-NEXT:    cbnz x9, .LBB0_2
+; CHECK-NEXT:  .LBB0_4: // %return
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_5:
-; CHECK-NEXT:    mov w0, #1 // =0x1
+; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ret
 entry:
   %cmp13 = icmp sgt i32 %c, 0
@@ -56,13 +59,13 @@ define i64 @IVIncHoist_not_all_user_in_header(i32 %c, ptr %a, ptr %b) {
 ; CHECK-LABEL: IVIncHoist_not_all_user_in_header:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    cmp w0, #1
-; CHECK-NEXT:    b.lt .LBB1_5
+; CHECK-NEXT:    b.lt .LBB1_6
 ; CHECK-NEXT:  // %bb.1: // %for.body.preheader
-; CHECK-NEXT:    mov x8, xzr
 ; CHECK-NEXT:    mov w9, w0
+; CHECK-NEXT:    mov x8, xzr
 ; CHECK-NEXT:    add x10, x1, #4
+; CHECK-NEXT:    sub x9, x9, #1
 ; CHECK-NEXT:    add x11, x2, #8
-; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:  .LBB1_2: // %for.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr w12, [x10, x8, lsl #2]
@@ -70,18 +73,22 @@ define i64 @IVIncHoist_not_all_user_in_header(i32 %c, ptr %a, ptr %b) {
 ; CHECK-NEXT:  // %bb.3: // %if.then
 ; CHECK-NEXT:    // in Loop: Header=BB1_2 Depth=1
 ; CHECK-NEXT:    ldr w12, [x11, x8, lsl #2]
-; CHECK-NEXT:    cbnz w12, .LBB1_6
-; CHECK-NEXT:  // %bb.4: // %for.cond
+; CHECK-NEXT:    add x13, x8, #3
+; CHECK-NEXT:    cmp w12, #0
+; CHECK-NEXT:    csel x0, xzr, x13, eq
+; CHECK-NEXT:    cbnz w12, .LBB1_5
+; CHECK-NEXT:  // %bb.4: // %if.then
 ; CHECK-NEXT:    // in Loop: Header=BB1_2 Depth=1
-; CHECK-NEXT:    add x8, x8, #1
 ; CHECK-NEXT:    cmp x9, x8
+; CHECK-NEXT:    add x8, x8, #1
 ; CHECK-NEXT:    b.ne .LBB1_2
-; CHECK-NEXT:  .LBB1_5:
+; CHECK-NEXT:  .LBB1_5: // %return
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB1_6:
 ; CHECK-NEXT:    mov x0, xzr
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB1_6: // %if.then.return.loopexit_crit_edge
-; CHECK-NEXT:    add x0, x8, #3
-; CHECK-NEXT:  .LBB1_7: // %return
+; CHECK-NEXT:  .LBB1_7:
+; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
 entry:
   %cmp13 = icmp sgt i32 %c, 0
@@ -126,23 +133,25 @@ define i32 @negative_test_type_is_struct(i32 %c, ptr %a, ptr %b) {
 ; CHECK-LABEL: negative_test_type_is_struct:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    cmp w0, #1
-; CHECK-NEXT:    b.lt .LBB2_4
+; CHECK-NEXT:    b.lt .LBB2_5
 ; CHECK-NEXT:  // %bb.1: // %for.body.preheader
 ; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    sub x8, x8, #1
 ; CHECK-NEXT:  .LBB2_2: // %for.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr w9, [x1]
-; CHECK-NEXT:    cbnz w9, .LBB2_5
-; CHECK-NEXT:  // %bb.3: // %for.cond
+; CHECK-NEXT:    ldr w10, [x1], #4
+; CHECK-NEXT:    cmp w10, #0
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    cbnz w10, .LBB2_4
+; CHECK-NEXT:  // %bb.3: // %for.body
 ; CHECK-NEXT:    // in Loop: Header=BB2_2 Depth=1
-; CHECK-NEXT:    subs x8, x8, #1
-; CHECK-NEXT:    add x1, x1, #4
-; CHECK-NEXT:    b.ne .LBB2_2
-; CHECK-NEXT:  .LBB2_4:
-; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    mov x9, x8
+; CHECK-NEXT:    sub x8, x8, #1
+; CHECK-NEXT:    cbnz x9, .LBB2_2
+; CHECK-NEXT:  .LBB2_4: // %return
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB2_5:
-; CHECK-NEXT:    mov w0, #1 // =0x1
+; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ret
 entry:
   %cmp13 = icmp sgt i32 %c, 0
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/small-constant.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/small-constant.ll
index cd6b410b67aa3..85670b9f39961 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/AArch64/small-constant.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/small-constant.ll
@@ -19,22 +19,23 @@ define float @test1(ptr nocapture readonly %arr, i64 %start, float %threshold) {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    cbz x1, .LBB0_4
 ; CHECK-NEXT:  // %bb.1: // %for.body.preheader
+; CHECK-NEXT:    fmov s2, #-7.00000000
 ; CHECK-NEXT:    add x8, x0, #28
 ; CHECK-NEXT:  .LBB0_2: // %for.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr s1, [x8, x1, lsl #2]
+; CHECK-NEXT:    adds x1, x1, #1
+; CHECK-NEXT:    cset w9, hs
 ; CHECK-NEXT:    fcmp s1, s0
-; CHECK-NEXT:    b.gt .LBB0_5
-; CHECK-NEXT:  // %bb.3: // %for.cond
-; CHECK-NEXT:    // in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT:    add x1, x1, #1
-; CHECK-NEXT:    cbnz x1, .LBB0_2
+; CHECK-NEXT:    fcsel s1, s1, s2, gt
+; CHECK-NEXT:    ccmp w9, #0, #0, le
+; CHECK-NEXT:    b.eq .LBB0_2
+; CHECK-NEXT:  // %bb.3: // %cleanup2
+; CHECK-NEXT:    fmov s0, s1
+; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_4:
 ; CHECK-NEXT:    fmov s0, #-7.00000000
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB0_5: // %cleanup2
-; CHECK-NEXT:    fmov s0, s1
-; CHECK-NEXT:    ret
 entry:
   %cmp11 = icmp eq i64 %start, 0
   br i1 %cmp11, label %cleanup2, label %for.body
@@ -64,24 +65,25 @@ define float @test2(ptr nocapture readonly %arr, i64 %start, float %threshold) {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    cbz x1, .LBB1_4
 ; CHECK-NEXT:  // %bb.1: // %for.body.preheader
+; CHECK-NEXT:    fmov s2, #-7.00000000
 ; CHECK-NEXT:    add x8, x0, #28
 ; CHECK-NEXT:  .LBB1_2: // %for.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    scvtf s1, x1
-; CHECK-NEXT:    fadd s2, s1, s0
-; CHECK-NEXT:    ldr s1, [x8, x1, lsl #2]
-; CHECK-NEXT:    fcmp s1, s2
-; CHECK-NEXT:    b.gt .LBB1_5
-; CHECK-NEXT:  // %bb.3: // %for.cond
-; CHECK-NEXT:    // in Loop: Header=BB1_2 Depth=1
-; CHECK-NEXT:    add x1, x1, #1
-; CHECK-NEXT:    cbnz x1, .LBB1_2
+; CHECK-NEXT:    ldr s3, [x8, x1, lsl #2]
+; CHECK-NEXT:    adds x1, x1, #1
+; CHECK-NEXT:    cset w9, hs
+; CHECK-NEXT:    fadd s1, s1, s0
+; CHECK-NEXT:    fcmp s3, s1
+; CHECK-NEXT:    fcsel s1, s3, s2, gt
+; CHECK-NEXT:    ccmp w9, #0, #0, le
+; CHECK-NEXT:    b.eq .LBB1_2
+; CHECK-NEXT:  // %bb.3: // %cleanup4
+; CHECK-NEXT:    fmov s0, s1
+; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB1_4:
 ; CHECK-NEXT:    fmov s0, #-7.00000000
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB1_5: // %cleanup4
-; CHECK-NEXT:    fmov s0, s1
-; CHECK-NEXT:    ret
 entry:
   %cmp14 = icmp eq i64 %start, 0
   br i1 %cmp14, label %cleanup4, label %for.body
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
index 5f6e66e344625..227c45fa80a1d 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
@@ -2967,14 +2967,9 @@ define void @non_loop(i32 %arg) {
 ; CHECK:       preheader:
 ; CHECK-NEXT:    br label %header
 ; CHECK:       header:
-; CHECK-NEXT:    br i1 true, label %latchExit, label %latch
-; CHECK:       latch:
-; CHECK-NEXT:    br label %latchExit
-; CHECK:       latchExit:
-; CHECK-NEXT:    %i2.ph = phi i32 [ %arg, %header ], [ -1, %latch ]
 ; CHECK-NEXT:    br label %returnblock
 ; CHECK:       returnblock:
-; CHECK-NEXT:    %i2 = phi i32 [ -1, %entry ], [ %i2.ph, %latchExit ]
+; CHECK-NEXT:    %i2 = phi i32 [ -1, %entry ], [ %arg, %header ]
 ; CHECK-NEXT:    ret void
 ;
 
@@ -3007,267 +3002,102 @@ returnblock:                                         ; preds = %latchExit, %entr
 define void @unique_exit(i32 %N, i32 %M) {
 ; EPILOG-LABEL: @unique_exit(
 ; EPILOG-NEXT:  preheader:
-; EPILOG-NEXT:    %M.shifted = shl i32 %M, 3
-; EPILOG-NEXT:    %umax = call i32 @llvm.umax.i32(i32 %M.shifted, i32 1)
-; EPILOG-NEXT:    %0 = freeze i32 %umax
-; EPILOG-NEXT:    %1 = add i32 %0, -1
-; EPILOG-NEXT:    %xtraiter = and i32 %0, 7
-; EPILOG-NEXT:    %2 = icmp ult i32 %1, 7
-; EPILOG-NEXT:    br i1 %2, label %latchExit.unr-lcssa, label %preheader.new
-; EPILOG:       preheader.new:
-; EPILOG-NEXT:    %unroll_iter = sub i32 %0, %xtraiter
+; EPILOG-NEXT:    %M.shifted = shl nuw i32 %M, 3
 ; EPILOG-NEXT:    br label %header
 ; EPILOG:       header:
-; EPILOG-NEXT:    %i4 = phi i32 [ 0, %preheader.new ], [ %inc.7, %latch.7 ]
-; EPILOG-NEXT:    %niter = phi i32 [ 0, %preheader.new ], [ %niter.next.7, %latch.7 ]
-; EPILOG-NEXT:    %inc = add nuw nsw i32 %i4, 1
+; EPILOG-NEXT:    %i4 = phi i32 [ 0, %preheader ], [ %inc, %header ]
+; EPILOG-NEXT:    %inc = add nuw i32 %i4, 1
 ; EPILOG-NEXT:    %cmp1 = icmp ult i32 %inc, %N
-; EPILOG-NEXT:    br i1 %cmp1, label %latch, label %latchExit.epilog-lcssa.loopexit
-; EPILOG:       latch:
-; EPILOG-NEXT:    %inc.1 = add nuw nsw i32 %i4, 2
-; EPILOG-NEXT:    %cmp1.1 = icmp ult i32 %inc.1, %N
-; EPILOG-NEXT:    br i1 %cmp1.1, label %latch.1, label %latchExit.epilog-lcssa.loopexit
-; EPILOG:       latch.1:
-; EPILOG-NEXT:    %inc.2 = add nuw nsw i32 %i4, 3
-; EPILOG-NEXT:    %cmp1.2 = icmp ult i32 %inc.2, %N
-; EPILOG-NEXT:    br i1 %cmp1.2, label %latch.2, label %latchExit.epilog-lcssa.loopexit
-; EPILOG:       latch.2:
-; EPILOG-NEXT:    %inc.3 = add nuw nsw i32 %i4, 4
-; EPILOG-NEXT:    %cmp1.3 = icmp ult i32 %inc.3, %N
-; EPILOG-NEXT:    br i1 %cmp1.3, label %latch.3, label %latchExit.epilog-lcssa.loopexit
-; EPILOG:       latch.3:
-; EPILOG-NEXT:    %inc.4 = add nuw nsw i32 %i4, 5
-; EPILOG-NEXT:    %cmp1.4 = icmp ult i32 %inc.4, %N
-; EPILOG-NEXT:    br i1 %cmp1.4, label %latch.4, label %latchExit.epilog-lcssa.loopexit
-; EPILOG:       latch.4:
-; EPILOG-NEXT:    %inc.5 = add nuw nsw i32 %i4, 6
-; EPILOG-NEXT:    %cmp1.5 = icmp ult i32 %inc.5, %N
-; EPILOG-NEXT:    br i1 %cmp1.5, label %latch.5, label %latchExit.epilog-lcssa.loopexit
-; EPILOG:       latch.5:
-; EPILOG-NEXT:    %inc.6 = add nuw nsw i32 %i4, 7
-; EPILOG-NEXT:    %cmp1.6 = icmp ult i32 %inc.6, %N
-; EPILOG-NEXT:    br i1 %cmp1.6, label %latch.6, label %latchExit.epilog-lcssa.loopexit
-; EPILOG:       latch.6:
-; EPILOG-NEXT:    %inc.7 = add nuw i32 %i4, 8
-; EPILOG-NEXT:    %cmp1.7 = icmp ult i32 %inc.7, %N
-; EPILOG-NEXT:    br i1 %cmp1.7, label %latch.7, label %latchExit.epilog-lcssa.loopexit
-; EPILOG:       latch.7:
-; EPILOG-NEXT:    %niter.next.7 = add nuw i32 %niter, 8
-; EPILOG-NEXT:    %niter.ncmp.7 = icmp ne i32 %niter.next.7, %unroll_iter
-; EPILOG-NEXT:    br i1 %niter.ncmp.7, label %header, label %latchExit.unr-lcssa.loopexit
-; EPILOG:       latchExit.unr-lcssa.loopexit:
-; EPILOG-NEXT:    %i2.ph.ph.ph = phi i32 [ -1, %latch.7 ]
-; EPILOG-NEXT:    %i4.unr.ph = phi i32 [ %inc.7, %latch.7 ]
-; EPILOG-NEXT:    br label %latchExit.unr-lcssa
-; EPILOG:       latchExit.unr-lcssa:
-; EPILOG-NEXT:    %i2.ph.ph = phi i32 [ poison, %preheader ], [ %i2.ph.ph.ph, %latchExit.unr-lcssa.loopexit ]
-; EPILOG-NEXT:    %i4.unr = phi i32 [ 0, %preheader ], [ %i4.unr.ph, %latchExit.unr-lcssa.loopexit ]
-; EPILOG-NEXT:    %lcmp.mod = icmp ne i32 %xtraiter, 0
-; EPILOG-NEXT:    br i1 %lcmp.mod, label %header.epil.preheader, label %latchExit
-; EPILOG:       header.epil.preheader:
-; EPILOG-NEXT:    br label %header.epil
-; EPILOG:       header.epil:
-; EPILOG-NEXT:    %i4.epil = phi i32 [ %inc.epil, %latch.epil ], [ %i4.unr, %header.epil.preheader ]
-; EPILOG-NEXT:    %epil.iter = phi i32 [ 0, %header.epil.preheader ], [ %epil.iter.next, %latch.epil ]
-; EPILOG-NEXT:    %inc.epil = add nuw i32 %i4.epil, 1
-; EPILOG-NEXT:    %cmp1.epil = icmp ult i32 %inc.epil, %N
-; EPILOG-NEXT:    br i1 %cmp1.epil, label %latch.epil, label %latchExit.epilog-lcssa.loopexit2
-; EPILOG:       latch.epil:
-; EPILOG-NEXT:    %cmp.epil = icmp ult i32 %inc.epil, %M.shifted
-; EPILOG-NEXT:    %epil.iter.next = add i32 %epil.iter, 1
-; EPILOG-NEXT:    %epil.iter.cmp = icmp ne i32 %epil.iter.next, %xtraiter
-; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %header.epil, label %latchExit.epilog-lcssa.loopexit2, !llvm.loop !8
-; EPILOG:       latchExit.epilog-lcssa.loopexit:
-; EPILOG-NEXT:    %i2.ph.ph1.ph = phi i32 [ %i4, %header ], [ %inc, %latch ], [ %inc.1, %latch.1 ], [ %inc.2, %latch.2 ], [ %inc.3, %latch.3 ], [ %inc.4, %latch.4 ], [ %inc.5, %latch.5 ], [ %inc.6, %latch.6 ]
-; EPILOG-NEXT:    br label %latchExit.epilog-lcssa
-; EPILOG:       latchExit.epilog-lcssa.loopexit2:
-; EPILOG-NEXT:    %i2.ph.ph1.ph3 = phi i32 [ %i4.epil, %header.epil ], [ -1, %latch.epil ]
-; EPILOG-NEXT:    br label %latchExit.epilog-lcssa
-; EPILOG:       latchExit.epilog-lcssa:
-; EPILOG-NEXT:    %i2.ph.ph1 = phi i32 [ %i2.ph.ph1.ph, %latchExit.epilog-lcssa.loopexit ], [ %i2.ph.ph1.ph3, %latchExit.epilog-lcssa.loopexit2 ]
-; EPILOG-NEXT:    br label %latchExit
+; EPILOG-NEXT:    %cmp = icmp ult i32 %inc, %M.shifted
+; EPILOG-NEXT:    %0 = select i1 %cmp1, i32 -1, i32 %i4
+; EPILOG-NEXT:    %or.cond = select i1 %cmp1, i1 %cmp, i1 false
+; EPILOG-NEXT:    br i1 %or.cond, label %header, label %latchExit
 ; EPILOG:       latchExit:
-; EPILOG-NEXT:    %i2.ph = phi i32 [ %i2.ph.ph, %latchExit.unr-lcssa ], [ %i2.ph.ph1, %latchExit.epilog-lcssa ]
 ; EPILOG-NEXT:    ret void
 ;
 ; EPILOG-BLOCK-LABEL: @unique_exit(
 ; EPILOG-BLOCK-NEXT:  preheader:
 ; EPILOG-BLOCK-NEXT:    %M.shifted = shl i32 %M, 3
 ; EPILOG-BLOCK-NEXT:    %umax = call i32 @llvm.umax.i32(i32 %M.shifted, i32 1)
-; EPILOG-BLOCK-NEXT:    %0 = freeze i32 %umax
-; EPILOG-BLOCK-NEXT:    %1 = add i32 %0, -1
-; EPILOG-BLOCK-NEXT:    %xtraiter = and i32 %0, 1
-; EPILOG-BLOCK-NEXT:    %2 = icmp ult i32 %1, 1
-; EPILOG-BLOCK-NEXT:    br i1 %2, label %latchExit.unr-lcssa, label %preheader.new
+; EPILOG-BLOCK-NEXT:    %0 = add i32 %umax, -1
+; EPILOG-BLOCK-NEXT:    %1 = freeze i32 %0
+; EPILOG-BLOCK-NEXT:    %umax1 = call i32 @llvm.umax.i32(i32 %N, i32 1)
+; EPILOG-BLOCK-NEXT:    %2 = add i32 %umax1, -1
+; EPILOG-BLOCK-NEXT:    %umin = call i32 @llvm.umin.i32(i32 %1, i32 %2)
+; EPILOG-BLOCK-NEXT:    %3 = add nuw i32 %umin, 1
+; EPILOG-BLOCK-NEXT:    %xtraiter = and i32 %3, 1
+; EPILOG-BLOCK-NEXT:    %4 = icmp ult i32 %umin, 1
+; EPILOG-BLOCK-NEXT:    br i1 %4, label %latchExit.unr-lcssa, label %preheader.new
 ; EPILOG-BLOCK:       preheader.new:
-; EPILOG-BLOCK-NEXT:    %unroll_iter = sub i32 %0, %xtraiter
+; EPILOG-BLOCK-NEXT:    %unroll_iter = sub i32 %3, %xtraiter
 ; EPILOG-BLOCK-NEXT:    br label %header
 ; EPILOG-BLOCK:       header:
-; EPILOG-BLOCK-NEXT:    %i4 = phi i32 [ 0, %preheader.new ], [ %inc.1, %latch.1 ]
-; EPILOG-BLOCK-NEXT:    %niter = phi i32 [ 0, %preheader.new ], [ %niter.next.1, %latch.1 ]
-; EPILOG-BLOCK-NEXT:    %inc = add nuw nsw i32 %i4, 1
-; EPILOG-BLOCK-NEXT:    %cmp1 = icmp ult i32 %inc, %N
-; EPILOG-BLOCK-NEXT:    br i1 %cmp1, label %latch, label %latchExit.epilog-lcssa.loopexit
-; EPILOG-BLOCK:       latch:
+; EPILOG-BLOCK-NEXT:    %i4 = phi i32 [ 0, %preheader.new ], [ %inc.1, %header ]
+; EPILOG-BLOCK-NEXT:    %niter = phi i32 [ 0, %preheader.new ], [ %niter.next.1, %header ]
 ; EPILOG-BLOCK-NEXT:    %inc.1 = add nuw i32 %i4, 2
-; EPILOG-BLOCK-NEXT:    %cmp1.1 = icmp ult i32 %inc.1, %N
-; EPILOG-BLOCK-NEXT:    br i1 %cmp1.1, label %latch.1, label %latchExit.epilog-lcssa.loopexit
-; EPILOG-BLOCK:       latch.1:
-; EPILOG-BLOCK-NEXT:    %niter.next.1 = add nuw i32 %niter, 2
+; EPILOG-BLOCK-NEXT:    %niter.next.1 = add i32 %niter, 2
 ; EPILOG-BLOCK-NEXT:    %niter.ncmp.1 = icmp ne i32 %niter.next.1, %unroll_iter
 ; EPILOG-BLOCK-NEXT:    br i1 %niter.ncmp.1, label %header, label %latchExit.unr-lcssa.loopexit, !llvm.loop !8
 ; EPILOG-BLOCK:       latchExit.unr-lcssa.loopexit:
-; EPILOG-BLOCK-NEXT:    %i2.ph.ph.ph = phi i32 [ -1, %latch.1 ]
-; EPILOG-BLOCK-NEXT:    %i4.unr.ph = phi i32 [ %inc.1, %latch.1 ]
 ; EPILOG-BLOCK-NEXT:    br label %latchExit.unr-lcssa
 ; EPILOG-BLOCK:       latchExit.unr-lcssa:
-; EPILOG-BLOCK-NEXT:    %i2.ph.ph = phi i32 [ poison, %preheader ], [ %i2.ph.ph.ph, %latchExit.unr-lcssa.loopexit ]
-; EPILOG-BLOCK-NEXT:    %i4.unr = phi i32 [ 0, %preheader ], [ %i4.unr.ph, %latchExit.unr-lcssa.loopexit ]
 ; EPILOG-BLOCK-NEXT:    %lcmp.mod = icmp ne i32 %xtraiter, 0
 ; EPILOG-BLOCK-NEXT:    br i1 %lcmp.mod, label %header.epil.preheader, label %latchExit
 ; EPILOG-BLOCK:       header.epil.preheader:
 ; EPILOG-BLOCK-NEXT:    br label %header.epil
 ; EPILOG-BLOCK:       header.epil:
-; EPILOG-BLOCK-NEXT:    %inc.epil = add nuw i32 %i4.unr, 1
-; EPILOG-BLOCK-NEXT:    %cmp1.epil = icmp ult i32 %inc.epil, %N
-; EPILOG-BLOCK-NEXT:    br i1 %cmp1.epil, label %latch.epil, label %latchExit.epilog-lcssa
-; EPILOG-BLOCK:       latch.epil:
-; EPILOG-BLOCK-NEXT:    br label %latchExit.epilog-lcssa
-; EPILOG-BLOCK:       latchExit.epilog-lcssa.loopexit:
-; EPILOG-BLOCK-NEXT:    %i2.ph.ph1.ph = phi i32 [ %i4, %header ], [ %inc, %latch ]
-; EPILOG-BLOCK-NEXT:    br label %latchExit.epilog-lcssa
-; EPILOG-BLOCK:       latchExit.epilog-lcssa:
-; EPILOG-BLOCK-NEXT:    %i2.ph.ph1 = phi i32 [ -1, %latch.epil ], [ %i4.unr, %header.epil ], [ %i2.ph.ph1.ph, %latchExit.epilog-lcssa.loopexit ]
 ; EPILOG-BLOCK-NEXT:    br label %latchExit
 ; EPILOG-BLOCK:       latchExit:
-; EPILOG-BLOCK-NEXT:    %i2.ph = phi i32 [ %i2.ph.ph, %latchExit.unr-lcssa ], [ %i2.ph.ph1, %latchExit.epilog-lcssa ]
 ; EPILOG-BLOCK-NEXT:    ret void
 ;
 ; PROLOG-LABEL: @unique_exit(
 ; PROLOG-NEXT:  preheader:
-; PROLOG-NEXT:    %M.shifted = shl i32 %M, 3
-; PROLOG-NEXT:    %umax = call i32 @llvm.umax.i32(i32 %M.shifted, i32 1)
-; PROLOG-NEXT:    %0 = freeze i32 %umax
-; PROLOG-NEXT:    %1 = add i32 %0, -1
-; PROLOG-NEXT:    %xtraiter = and i32 %0, 7
-; PROLOG-NEXT:    %lcmp.mod = icmp ne i32 %xtraiter, 0
-; PROLOG-NEXT:    br i1 %lcmp.mod, label %header.prol.preheader, label %header.prol.loopexit
-; PROLOG:       header.prol.preheader:
-; PROLOG-NEXT:    br label %header.prol
-; PROLOG:       header.prol:
-; PROLOG-NEXT:    %i4.prol = phi i32 [ %inc.prol, %latch.prol ], [ 0, %header.prol.preheader ]
-; PROLOG-NEXT:    %prol.iter = phi i32 [ 0, %header.prol.preheader ], [ %prol.iter.next, %latch.prol ]
-; PROLOG-NEXT:    %inc.prol = add nuw i32 %i4.prol, 1
-; PROLOG-NEXT:    %cmp1.prol = icmp ult i32 %inc.prol, %N
-; PROLOG-NEXT:    br i1 %cmp1.prol, label %latch.prol, label %latchExit.unr-lcssa.loopexit1
-; PROLOG:       latch.prol:
-; PROLOG-NEXT:    %cmp.prol = icmp ult i32 %inc.prol, %M.shifted
-; PROLOG-NEXT:    %prol.iter.next = add i32 %prol.iter, 1
-; PROLOG-NEXT:    %prol.iter.cmp = icmp ne i32 %prol.iter.next, %xtraiter
-; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %header.prol, label %header.prol.loopexit.unr-lcssa, !llvm.loop !8
-; PROLOG:       header.prol.loopexit.unr-lcssa:
-; PROLOG-NEXT:    %i4.unr.ph = phi i32 [ %inc.prol, %latch.prol ]
-; PROLOG-NEXT:    %i2.ph.unr.ph = phi i32 [ -1, %latch.prol ]
-; PROLOG-NEXT:    br label %header.prol.loopexit
-; PROLOG:       header.prol.loopexit:
-; PROLOG-NEXT:    %i4.unr = phi i32 [ 0, %preheader ], [ %i4.unr.ph, %header.prol.loopexit.unr-lcssa ]
-; PROLOG-NEXT:    %i2.ph.unr = phi i32 [ poison, %preheader ], [ %i2.ph.unr.ph, %header.prol.loopexit.unr-lcssa ]
-; PROLOG-NEXT:    %2 = icmp ult i32 %1, 7
-; PROLOG-NEXT:    br i1 %2, label %latchExit, label %preheader.new
-; PROLOG:       preheader.new:
+; PROLOG-NEXT:    %M.shifted = shl nuw i32 %M, 3
 ; PROLOG-NEXT:    br label %header
 ; PROLOG:       header:
-; PROLOG-NEXT:    %i4 = phi i32 [ %i4.unr, %preheader.new ], [ %inc.7, %latch.7 ]
+; PROLOG-NEXT:    %i4 = phi i32 [ 0, %preheader ], [ %inc, %header ]
 ; PROLOG-NEXT:    %inc = add nuw i32 %i4, 1
 ; PROLOG-NEXT:    %cmp1 = icmp ult i32 %inc, %N
-; PROLOG-NEXT:    br i1 %cmp1, label %latch, label %latchExit.unr-lcssa.loopexit
-; PROLOG:       latch:
-; PROLOG-NEXT:    %inc.1 = add nuw i32 %i4, 2
-; PROLOG-NEXT:    %cmp1.1 = icmp ult i32 %inc.1, %N
-; PROLOG-NEXT:    br i1 %cmp1.1, label %latch.1, label %latchExit.unr-lcssa.loopexit
-; PROLOG:       latch.1:
-; PROLOG-NEXT:    %inc.2 = add nuw i32 %i4, 3
-; PROLOG-NEXT:    %cmp1.2 = icmp ult i32 %inc.2, %N
-; PROLOG-NEXT:    br i1 %cmp1.2, label %latch.2, label %latchExit.unr-lcssa.loopexit
-; PROLOG:       latch.2:
-; PROLOG-NEXT:    %inc.3 = add nuw i32 %i4, 4
-; PROLOG-NEXT:    %cmp1.3 = icmp ult i32 %inc.3, %N
-; PROLOG-NEXT:    br i1 %cmp1.3, label %latch.3, label %latchExit.unr-lcssa.loopexit
-; PROLOG:       latch.3:
-; PROLOG-NEXT:    %inc.4 = add nuw i32 %i4, 5
-; PROLOG-NEXT:    %cmp1.4 = icmp ult i32 %inc.4, %N
-; PROLOG-NEXT:    br i1 %cmp1.4, label %latch.4, label %latchExit.unr-lcssa.loopexit
-; PROLOG:       latch.4:
-; PROLOG-NEXT:    %inc.5 = add nuw i32 %i4, 6
-; PROLOG-NEXT:    %cmp1.5 = icmp ult i32 %inc.5, %N
-; PROLOG-NEXT:    br i1 %cmp1.5, label %latch.5, label %latchExit.unr-lcssa.loopexit
-; PROLOG:       latch.5:
-; PROLOG-NEXT:    %inc.6 = add nuw i32 %i4, 7
-; PROLOG-NEXT:    %cmp1.6 = icmp ult i32 %inc.6, %N
-; PROLOG-NEXT:    br i1 %cmp1.6, label %latch.6, label %latchExit.unr-lcssa.loopexit
-; PROLOG:       latch.6:
-; PROLOG-NEXT:    %inc.7 = add nuw i32 %i4, 8
-; PROLOG-NEXT:    %cmp1.7 = icmp ult i32 %inc.7, %N
-; PROLOG-NEXT:    br i1 %cmp1.7, label %latch.7, label %latchExit.unr-lcssa.loopexit
-; PROLOG:       latch.7:
-; PROLOG-NEXT:    %cmp.7 = icmp ult i32 %inc.7, %M.shifted
-; PROLOG-NEXT:    br i1 %cmp.7, label %header, label %latchExit.unr-lcssa.loopexit
-; PROLOG:       latchExit.unr-lcssa.loopexit:
-; PROLOG-NEXT:    %i2.ph.ph.ph = phi i32 [ %i4, %header ], [ %inc, %latch ], [ %inc.1, %latch.1 ], [ %inc.2, %latch.2 ], [ %inc.3, %latch.3 ], [ %inc.4, %latch.4 ], [ %inc.5, %latch.5 ], [ %inc.6, %latch.6 ], [ -1, %latch.7 ]
-; PROLOG-NEXT:    br label %latchExit.unr-lcssa
-; PROLOG:       latchExit.unr-lcssa.loopexit1:
-; PROLOG-NEXT:    %i2.ph.ph.ph2 = phi i32 [ %i4.prol, %header.prol ]
-; PROLOG-NEXT:    br label %latchExit.unr-lcssa
-; PROLOG:       latchExit.unr-lcssa:
-; PROLOG-NEXT:    %i2.ph.ph = phi i32 [ %i2.ph.ph.ph, %latchExit.unr-lcssa.loopexit ], [ %i2.ph.ph.ph2, %latchExit.unr-lcssa.loopexit1 ]
-; PROLOG-NEXT:    br label %latchExit
+; PROLOG-NEXT:    %cmp = icmp ult i32 %inc, %M.shifted
+; PROLOG-NEXT:    %0 = select i1 %cmp1, i32 -1, i32 %i4
+; PROLOG-NEXT:    %or.cond = select i1 %cmp1, i1 %cmp, i1 false
+; PROLOG-NEXT:    br i1 %or.cond, label %header, label %latchExit
 ; PROLOG:       latchExit:
-; PROLOG-NEXT:    %i2.ph = phi i32 [ %i2.ph.unr, %header.prol.loopexit ], [ %i2.ph.ph, %latchExit.unr-lcssa ]
 ; PROLOG-NEXT:    ret void
 ;
 ; PROLOG-BLOCK-LABEL: @unique_exit(
 ; PROLOG-BLOCK-NEXT:  preheader:
 ; PROLOG-BLOCK-NEXT:    %M.shifted = shl i32 %M, 3
 ; PROLOG-BLOCK-NEXT:    %umax = call i32 @llvm.umax.i32(i32 %M.shifted, i32 1)
-; PROLOG-BLOCK-NEXT:    %0 = freeze i32 %umax
-; PROLOG-BLOCK-NEXT:    %1 = add i32 %0, -1
-; PROLOG-BLOCK-NEXT:    %xtraiter = and i32 %0, 1
+; PROLOG-BLOCK-NEXT:    %0 = add i32 %umax, -1
+; PROLOG-BLOCK-NEXT:    %1 = freeze i32 %0
+; PROLOG-BLOCK-NEXT:    %umax1 = call i32 @llvm.umax.i32(i32 %N, i32 1)
+; PROLOG-BLOCK-NEXT:    %2 = add i32 %umax1, -1
+; PROLOG-BLOCK-NEXT:    %umin = call i32 @llvm.umin.i32(i32 %1, i32 %2)
+; PROLOG-BLOCK-NEXT:    %3 = add nuw i32 %umin, 1
+; PROLOG-BLOCK-NEXT:    %xtraiter = and i32 %3, 1
 ; PROLOG-BLOCK-NEXT:    %lcmp.mod = icmp ne i32 %xtraiter, 0
 ; PROLOG-BLOCK-NEXT:    br i1 %lcmp.mod, label %header.prol.preheader, label %header.prol.loopexit
 ; PROLOG-BLOCK:       header.prol.preheader:
 ; PROLOG-BLOCK-NEXT:    br label %header.prol
 ; PROLOG-BLOCK:       header.prol:
-; PROLOG-BLOCK-NEXT:    %cmp1.prol = icmp ult i32 1, %N
-; PROLOG-BLOCK-NEXT:    br i1 %cmp1.prol, label %latch.prol, label %latchExit.unr-lcssa
-; PROLOG-BLOCK:       latch.prol:
 ; PROLOG-BLOCK-NEXT:    br label %header.prol.loopexit
 ; PROLOG-BLOCK:       header.prol.loopexit:
-; PROLOG-BLOCK-NEXT:    %i4.unr = phi i32 [ 0, %preheader ], [ 1, %latch.prol ]
-; PROLOG-BLOCK-NEXT:    %i2.ph.unr = phi i32 [ poison, %preheader ], [ -1, %latch.prol ]
-; PROLOG-BLOCK-NEXT:    %2 = icmp ult i32 %1, 1
-; PROLOG-BLOCK-NEXT:    br i1 %2, label %latchExit, label %preheader.new
+; PROLOG-BLOCK-NEXT:    %i4.unr = phi i32 [ 0, %preheader ], [ 1, %header.prol ]
+; PROLOG-BLOCK-NEXT:    %4 = icmp ult i32 %umin, 1
+; PROLOG-BLOCK-NEXT:    br i1 %4, label %latchExit, label %preheader.new
 ; PROLOG-BLOCK:       preheader.new:
 ; PROLOG-BLOCK-NEXT:    br label %header
 ; PROLOG-BLOCK:       header:
-; PROLOG-BLOCK-NEXT:    %i4 = phi i32 [ %i4.unr, %preheader.new ], [ %inc.1, %latch.1 ]
-; PROLOG-BLOCK-NEXT:    %inc = add nuw i32 %i4, 1
-; PROLOG-BLOCK-NEXT:    %cmp1 = icmp ult i32 %inc, %N
-; PROLOG-BLOCK-NEXT:    br i1 %cmp1, label %latch, label %latchExit.unr-lcssa.loopexit
-; PROLOG-BLOCK:       latch:
+; PROLOG-BLOCK-NEXT:    %i4 = phi i32 [ %i4.unr, %preheader.new ], [ %inc.1, %header ]
 ; PROLOG-BLOCK-NEXT:    %inc.1 = add nuw i32 %i4, 2
 ; PROLOG-BLOCK-NEXT:    %cmp1.1 = icmp ult i32 %inc.1, %N
-; PROLOG-BLOCK-NEXT:    br i1 %cmp1.1, label %latch.1, label %latchExit.unr-lcssa.loopexit
-; PROLOG-BLOCK:       latch.1:
 ; PROLOG-BLOCK-NEXT:    %cmp.1 = icmp ult i32 %inc.1, %M.shifted
-; PROLOG-BLOCK-NEXT:    br i1 %cmp.1, label %header, label %latchExit.unr-lcssa.loopexit, !llvm.loop !8
-; PROLOG-BLOCK:       latchExit.unr-lcssa.loopexit:
-; PROLOG-BLOCK-NEXT:    %i2.ph.ph.ph = phi i32 [ %i4, %header ], [ %inc, %latch ], [ -1, %latch.1 ]
-; PROLOG-BLOCK-NEXT:    br label %latchExit.unr-lcssa
+; PROLOG-BLOCK-NEXT:    %or.cond.1 = select i1 %cmp1.1, i1 %cmp.1, i1 false
+; PROLOG-BLOCK-NEXT:    br i1 %or.cond.1, label %header, label %latchExit.unr-lcssa, !llvm.loop !8
 ; PROLOG-BLOCK:       latchExit.unr-lcssa:
-; PROLOG-BLOCK-NEXT:    %i2.ph.ph = phi i32 [ 0, %header.prol ], [ %i2.ph.ph.ph, %latchExit.unr-lcssa.loopexit ]
 ; PROLOG-BLOCK-NEXT:    br label %latchExit
 ; PROLOG-BLOCK:       latchExit:
-; PROLOG-BLOCK-NEXT:    %i2.ph = phi i32 [ %i2.ph.unr, %header.prol.loopexit ], [ %i2.ph.ph, %latchExit.unr-lcssa ]
 ; PROLOG-BLOCK-NEXT:    ret void
 ;
 
@@ -3430,7 +3260,7 @@ define i64 @test5(i64 %trip, i64 %add, i1 %cond) {
 ; EPILOG-NEXT:    %cmp.epil = icmp ne i64 %iv_next.epil, %trip
 ; EPILOG-NEXT:    %epil.iter.next = add i64 %epil.iter, 1
 ; EPILOG-NEXT:    %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter
-; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %loop_header.epil, label %latchexit.epilog-lcssa, !llvm.loop !9
+; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %loop_header.epil, label %latchexit.epilog-lcssa, !llvm.loop !8
 ; EPILOG:       latchexit.epilog-lcssa:
 ; EPILOG-NEXT:    %sum.next.lcssa.ph1 = phi i64 [ %sum.next.epil, %loop_latch.epil ]
 ; EPILOG-NEXT:    br label %latchexit
@@ -3535,7 +3365,7 @@ define i64 @test5(i64 %trip, i64 %add, i1 %cond) {
 ; PROLOG-NEXT:    %cmp.prol = icmp ne i64 %iv_next.prol, %trip
 ; PROLOG-NEXT:    %prol.iter.next = add i64 %prol.iter, 1
 ; PROLOG-NEXT:    %prol.iter.cmp = icmp ne i64 %prol.iter.next, %xtraiter
-; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol, label %loop_header.prol.loopexit.unr-lcssa, !llvm.loop !9
+; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol, label %loop_header.prol.loopexit.unr-lcssa, !llvm.loop !8
 ; PROLOG:       loop_header.prol.loopexit.unr-lcssa:
 ; PROLOG-NEXT:    %iv.unr.ph = phi i64 [ %iv_next.prol, %loop_latch.prol ]
 ; PROLOG-NEXT:    %sum.unr.ph = phi i64 [ %sum.next.prol, %loop_latch.prol ]
@@ -3864,7 +3694,7 @@ define i32 @test6(ptr nocapture %a, i64 %n, i1 %cond, i32 %x) {
 ; EPILOG-NEXT:    %exitcond.epil = icmp eq i64 %indvars.iv.next.epil, %n
 ; EPILOG-NEXT:    %epil.iter.next = add i64 %epil.iter, 1
 ; EPILOG-NEXT:    %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter
-; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %header.epil, label %latch_exit.epilog-lcssa, !llvm.loop !10
+; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %header.epil, label %latch_exit.epilog-lcssa, !llvm.loop !9
 ; EPILOG:       latch_exit.epilog-lcssa:
 ; EPILOG-NEXT:    %sum.0.lcssa.ph1 = phi i32 [ %add.epil, %latch.epil ]
 ; EPILOG-NEXT:    br label %latch_exit
@@ -3984,7 +3814,7 @@ define i32 @test6(ptr nocapture %a, i64 %n, i1 %cond, i32 %x) {
 ; PROLOG-NEXT:    %exitcond.prol = icmp eq i64 %indvars.iv.next.prol, %n
 ; PROLOG-NEXT:    %prol.iter.next = add i64 %prol.iter, 1
 ; PROLOG-NEXT:    %prol.iter.cmp = icmp ne i64 %prol.iter.next, %xtraiter
-; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %header.prol, label %header.prol.loopexit.unr-lcssa, !llvm.loop !10
+; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %header.prol, label %header.prol.loopexit.unr-lcssa, !llvm.loop !9
 ; PROLOG:       header.prol.loopexit.unr-lcssa:
 ; PROLOG-NEXT:    %sum.0.lcssa.unr.ph = phi i32 [ %add.prol, %latch.prol ]
 ; PROLOG-NEXT:    %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.prol, %latch.prol ]
@@ -4258,7 +4088,7 @@ define i32 @test7(i32 %arg, i32 %arg1, i32 %arg2) {
 ; EPILOG-NEXT:    %i9.epil = icmp slt i64 %add.epil, %sext
 ; EPILOG-NEXT:    %epil.iter.next = add i64 %epil.iter, 1
 ; EPILOG-NEXT:    %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter
-; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %header.epil, label %latchexit.epilog-lcssa, !llvm.loop !11
+; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %header.epil, label %latchexit.epilog-lcssa, !llvm.loop !10
 ; EPILOG:       latchexit.epilog-lcssa:
 ; EPILOG-NEXT:    br label %latchexit
 ; EPILOG:       latchexit:
@@ -4348,7 +4178,7 @@ define i32 @test7(i32 %arg, i32 %arg1, i32 %arg2) {
 ; PROLOG-NEXT:    %i9.prol = icmp slt i64 %add.prol, %sext
 ; PROLOG-NEXT:    %prol.iter.next = add i64 %prol.iter, 1
 ; PROLOG-NEXT:    %prol.iter.cmp = icmp ne i64 %prol.iter.next, %xtraiter
-; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %header.prol, label %header.prol.loopexit.unr-lcssa, !llvm.loop !11
+; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %header.prol, label %header.prol.loopexit.unr-lcssa, !llvm.loop !10
 ; PROLOG:       header.prol.loopexit.unr-lcssa:
 ; PROLOG-NEXT:    %i6.unr.ph = phi i64 [ %add.prol, %latch.prol ]
 ; PROLOG-NEXT:    br label %header.prol.loopexit
@@ -4535,7 +4365,7 @@ define void @test8() {
 ; EPILOG-NEXT:    %i6.epil = icmp ult i64 %i4.epil, 100
 ; EPILOG-NEXT:    %epil.iter.next = add i64 %epil.iter, 1
 ; EPILOG-NEXT:    %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter
-; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %innerH.epil, label %exit.epilog-lcssa, !llvm.loop !12
+; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %innerH.epil, label %exit.epilog-lcssa, !llvm.loop !11
 ; EPILOG:       exit.epilog-lcssa:
 ; EPILOG-NEXT:    br label %exit
 ; EPILOG:       exit.loopexit:
@@ -4639,7 +4469,7 @@ define void @test8() {
 ; PROLOG-NEXT:    %i6.prol = icmp ult i64 %i4.prol, 100
 ; PROLOG-NEXT:    %prol.iter.next = add i64 %prol.iter, 1
 ; PROLOG-NEXT:    %prol.iter.cmp = icmp ne i64 %prol.iter.next, %xtraiter
-; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %innerH.prol, label %innerH.prol.loopexit.unr-lcssa, !llvm.loop !12
+; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %innerH.prol, label %innerH.prol.loopexit.unr-lcssa, !llvm.loop !11
 ; PROLOG:       innerH.prol.loopexit.unr-lcssa:
 ; PROLOG-NEXT:    %i3.unr.ph = phi i64 [ %i4.prol, %latch.prol ]
 ; PROLOG-NEXT:    br label %innerH.prol.loopexit
@@ -4845,7 +4675,7 @@ define ptr addrspace(1) @test9(ptr nocapture readonly %arg, i32 %n) {
 ; EPILOG-NEXT:    %iv.next.epil = add nuw nsw i64 %phi.epil, 1
 ; EPILOG-NEXT:    %epil.iter.next = add i32 %epil.iter, 1
 ; EPILOG-NEXT:    %epil.iter.cmp = icmp ne i32 %epil.iter.next, %xtraiter
-; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %header.epil, label %outerLatch.loopexit.epilog-lcssa, !llvm.loop !13
+; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %header.epil, label %outerLatch.loopexit.epilog-lcssa, !llvm.loop !12
 ; EPILOG:       outerLatch.loopexit.epilog-lcssa:
 ; EPILOG-NEXT:    br label %outerLatch.loopexit
 ; EPILOG:       outerLatch.loopexit:
@@ -4978,7 +4808,7 @@ define ptr addrspace(1) @test9(ptr nocapture readonly %arg, i32 %n) {
 ; PROLOG-NEXT:    %iv.next.prol = add nuw nsw i64 %phi.prol, 1
 ; PROLOG-NEXT:    %prol.iter.next = add i32 %prol.iter, 1
 ; PROLOG-NEXT:    %prol.iter.cmp = icmp ne i32 %prol.iter.next, %xtraiter
-; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %header.prol, label %header.prol.loopexit.unr-lcssa, !llvm.loop !13
+; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %header.prol, label %header.prol.loopexit.unr-lcssa, !llvm.loop !12
 ; PROLOG:       header.prol.loopexit.unr-lcssa:
 ; PROLOG-NEXT:    %phi.unr.ph = phi i64 [ %iv.next.prol, %latch.prol ]
 ; PROLOG-NEXT:    br label %header.prol.loopexit
@@ -5247,7 +5077,7 @@ define void @test10(i64 %trip, i64 %trip2) {
 ; EPILOG-NEXT:    %cmp.epil = icmp ne i64 %iv_next.epil, %trip
 ; EPILOG-NEXT:    %epil.iter.next = add i64 %epil.iter, 1
 ; EPILOG-NEXT:    %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter
-; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %loop_header.epil, label %exit2.epilog-lcssa, !llvm.loop !14
+; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %loop_header.epil, label %exit2.epilog-lcssa, !llvm.loop !13
 ; EPILOG:       exit2.epilog-lcssa:
 ; EPILOG-NEXT:    br label %exit2
 ; EPILOG:       exit2:
@@ -5321,7 +5151,7 @@ define void @test10(i64 %trip, i64 %trip2) {
 ; PROLOG-NEXT:    %cmp.prol = icmp ne i64 %iv_next.prol, %trip
 ; PROLOG-NEXT:    %prol.iter.next = add i64 %prol.iter, 1
 ; PROLOG-NEXT:    %prol.iter.cmp = icmp ne i64 %prol.iter.next, %xtraiter
-; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol, label %loop_header.prol.loopexit.unr-lcssa, !llvm.loop !14
+; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol, label %loop_header.prol.loopexit.unr-lcssa, !llvm.loop !13
 ; PROLOG:       loop_header.prol.loopexit.unr-lcssa:
 ; PROLOG-NEXT:    %iv.unr.ph = phi i64 [ %iv_next.prol, %loop_latch.prol ]
 ; PROLOG-NEXT:    br label %loop_header.prol.loopexit
@@ -5520,7 +5350,7 @@ define void @test11(i64 %trip, i1 %cond) {
 ; EPILOG-NEXT:    %cmp.epil = icmp ne i64 %iv_next.epil, %trip
 ; EPILOG-NEXT:    %epil.iter.next = add i64 %epil.iter, 1
 ; EPILOG-NEXT:    %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter
-; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %loop_header.epil, label %exit2.epilog-lcssa, !llvm.loop !15
+; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %loop_header.epil, label %exit2.epilog-lcssa, !llvm.loop !14
 ; EPILOG:       exit2.epilog-lcssa:
 ; EPILOG-NEXT:    br label %exit2
 ; EPILOG:       exit2:
@@ -5587,7 +5417,7 @@ define void @test11(i64 %trip, i1 %cond) {
 ; PROLOG-NEXT:    %cmp.prol = icmp ne i64 %iv_next.prol, %trip
 ; PROLOG-NEXT:    %prol.iter.next = add i64 %prol.iter, 1
 ; PROLOG-NEXT:    %prol.iter.cmp = icmp ne i64 %prol.iter.next, %xtraiter
-; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol, label %loop_header.prol.loopexit.unr-lcssa, !llvm.loop !15
+; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol, label %loop_header.prol.loopexit.unr-lcssa, !llvm.loop !14
 ; PROLOG:       loop_header.prol.loopexit.unr-lcssa:
 ; PROLOG-NEXT:    %iv.unr.ph = phi i64 [ %iv_next.prol, %loop_latch.prol ]
 ; PROLOG-NEXT:    br label %loop_header.prol.loopexit
@@ -5794,7 +5624,7 @@ define void @test12(i64 %trip, i64 %trip2, i1 %cond) {
 ; EPILOG-NEXT:    %cmp.epil = icmp ne i64 %iv_next.epil, %trip
 ; EPILOG-NEXT:    %epil.iter.next = add i64 %epil.iter, 1
 ; EPILOG-NEXT:    %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter
-; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %loop_header.epil, label %exit1.epilog-lcssa.loopexit1, !llvm.loop !16
+; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %loop_header.epil, label %exit1.epilog-lcssa.loopexit1, !llvm.loop !15
 ; EPILOG:       exit1.epilog-lcssa.loopexit:
 ; EPILOG-NEXT:    br label %exit1.epilog-lcssa
 ; EPILOG:       exit1.epilog-lcssa.loopexit1:
@@ -5880,7 +5710,7 @@ define void @test12(i64 %trip, i64 %trip2, i1 %cond) {
 ; PROLOG-NEXT:    %cmp.prol = icmp ne i64 %iv_next.prol, %trip
 ; PROLOG-NEXT:    %prol.iter.next = add i64 %prol.iter, 1
 ; PROLOG-NEXT:    %prol.iter.cmp = icmp ne i64 %prol.iter.next, %xtraiter
-; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol, label %loop_header.prol.loopexit.unr-lcssa, !llvm.loop !16
+; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol, label %loop_header.prol.loopexit.unr-lcssa, !llvm.loop !15
 ; PROLOG:       loop_header.prol.loopexit.unr-lcssa:
 ; PROLOG-NEXT:    %iv.unr.ph = phi i64 [ %iv_next.prol, %loop_latch.prol ]
 ; PROLOG-NEXT:    br label %loop_header.prol.loopexit
@@ -6135,7 +5965,7 @@ define void @test13(i64 %trip, i64 %trip2) {
 ; EPILOG-NEXT:    %cmp.epil = icmp ne i64 %iv_next.epil, %trip
 ; EPILOG-NEXT:    %epil.iter.next = add i64 %epil.iter, 1
 ; EPILOG-NEXT:    %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter
-; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %loop_header.epil, label %exit1.epilog-lcssa.loopexit1, !llvm.loop !17
+; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %loop_header.epil, label %exit1.epilog-lcssa.loopexit1, !llvm.loop !16
 ; EPILOG:       exit1.epilog-lcssa.loopexit:
 ; EPILOG-NEXT:    br label %exit1.epilog-lcssa
 ; EPILOG:       exit1.epilog-lcssa.loopexit1:
@@ -6225,7 +6055,7 @@ define void @test13(i64 %trip, i64 %trip2) {
 ; PROLOG-NEXT:    %cmp.prol = icmp ne i64 %iv_next.prol, %trip
 ; PROLOG-NEXT:    %prol.iter.next = add i64 %prol.iter, 1
 ; PROLOG-NEXT:    %prol.iter.cmp = icmp ne i64 %prol.iter.next, %xtraiter
-; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol, label %loop_header.prol.loopexit.unr-lcssa, !llvm.loop !17
+; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol, label %loop_header.prol.loopexit.unr-lcssa, !llvm.loop !16
 ; PROLOG:       loop_header.prol.loopexit.unr-lcssa:
 ; PROLOG-NEXT:    %iv.unr.ph = phi i64 [ %iv_next.prol, %loop_latch.prol ]
 ; PROLOG-NEXT:    br label %loop_header.prol.loopexit
@@ -6474,7 +6304,7 @@ define void @test14(i64 %trip, i1 %cond) {
 ; EPILOG-NEXT:    %cmp.epil = icmp ne i64 %iv_next.epil, %trip
 ; EPILOG-NEXT:    %epil.iter.next = add i64 %epil.iter, 1
 ; EPILOG-NEXT:    %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter
-; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %loop_header.epil, label %exit1.epilog-lcssa.loopexit1, !llvm.loop !18
+; EPILOG-NEXT:    br i1 %epil.iter.cmp, label %loop_header.epil, label %exit1.epilog-lcssa.loopexit1, !llvm.loop !17
 ; EPILOG:       exit1.epilog-lcssa.loopexit:
 ; EPILOG-NEXT:    br label %exit1.epilog-lcssa
 ; EPILOG:       exit1.epilog-lcssa.loopexit1:
@@ -6557,7 +6387,7 @@ define void @test14(i64 %trip, i1 %cond) {
 ; PROLOG-NEXT:    %cmp.prol = icmp ne i64 %iv_next.prol, %trip
 ; PROLOG-NEXT:    %prol.iter.next = add i64 %prol.iter, 1
 ; PROLOG-NEXT:    %prol.iter.cmp = icmp ne i64 %prol.iter.next, %xtraiter
-; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol, label %loop_header.prol.loopexit.unr-lcssa, !llvm.loop !18
+; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol, label %loop_header.prol.loopexit.unr-lcssa, !llvm.loop !17
 ; PROLOG:       loop_header.prol.loopexit.unr-lcssa:
 ; PROLOG-NEXT:    %iv.unr.ph = phi i64 [ %iv_next.prol, %loop_latch.prol ]
 ; PROLOG-NEXT:    br label %loop_header.prol.loopexit
diff --git a/llvm/test/Transforms/LoopUnroll/scevunroll.ll b/llvm/test/Transforms/LoopUnroll/scevunroll.ll
index b6b14e365cc1d..aa33b979bf17e 100644
--- a/llvm/test/Transforms/LoopUnroll/scevunroll.ll
+++ b/llvm/test/Transforms/LoopUnroll/scevunroll.ll
@@ -310,16 +310,16 @@ define void @nsw_latch(ptr %a) nounwind {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    br label [[FOR_COND:%.*]]
-; CHECK:       for.cond:
-; CHECK-NEXT:    br i1 false, label [[RETURN:%.*]], label [[FOR_BODY_1:%.*]]
-; CHECK:       for.body.1:
-; CHECK-NEXT:    br i1 false, label [[FOR_COND_1:%.*]], label [[RETURN]]
-; CHECK:       for.cond.1:
-; CHECK-NEXT:    br label [[RETURN]]
+; CHECK-NEXT:    [[B_03:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[B_03]], 0
+; CHECK-NEXT:    [[ADD]] = add nuw nsw i32 [[B_03]], 8
+; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = xor i1 [[TOBOOL]], true
+; CHECK-NEXT:    [[TMP0:%.*]] = select i1 [[TOBOOL]], i32 [[B_03]], i32 [[B_03]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[TOBOOL]], i32 0, i32 1
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[TOBOOL_NOT]], i1 true, i1 false
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[RETURN:%.*]], label [[FOR_BODY]]
 ; CHECK:       return:
-; CHECK-NEXT:    [[B_03_LCSSA:%.*]] = phi i32 [ 0, [[FOR_COND]] ], [ 8, [[FOR_BODY_1]] ], [ 0, [[FOR_COND_1]] ]
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ 0, [[FOR_COND]] ], [ 1, [[FOR_BODY_1]] ], [ 0, [[FOR_COND_1]] ]
+; CHECK-NEXT:    [[B_03_LCSSA:%.*]] = phi i32 [ [[TMP0]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    store i32 [[B_03_LCSSA]], ptr [[A:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-after-peel.ll b/llvm/test/Transforms/LoopUnroll/unroll-after-peel.ll
index 504abafd23bf5..a1af3593e67b6 100644
--- a/llvm/test/Transforms/LoopUnroll/unroll-after-peel.ll
+++ b/llvm/test/Transforms/LoopUnroll/unroll-after-peel.ll
@@ -8,10 +8,10 @@ define i64 @hoge(i1 %c) {
 ; CHECK:       bb1.peel.begin:
 ; CHECK-NEXT:    br label [[BB1_PEEL:%.*]]
 ; CHECK:       bb1.peel:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB2_PEEL:%.*]], label [[BB4:%.*]]
-; CHECK:       bb2.peel:
 ; CHECK-NEXT:    [[TMP3_PEEL:%.*]] = icmp slt i32 0, 9
-; CHECK-NEXT:    br i1 [[TMP3_PEEL]], label [[BB1_PEEL_NEXT:%.*]], label [[BB4]]
+; CHECK-NEXT:    %0 = select i1 [[C:%.*]], i32 8, i32 0
+; CHECK-NEXT:    [[OR_COND_PEEL:%.*]] = and i1 [[C]], [[TMP3_PEEL]]
+; CHECK-NEXT:    br i1 [[OR_COND_PEEL]], label [[BB1_PEEL_NEXT:%.*]], label [[BB4:%.*]]
 ; CHECK:       bb1.peel.next:
 ; CHECK-NEXT:    br label [[BB1_PEEL_NEXT1:%.*]]
 ; CHECK:       bb1.peel.next1:
@@ -21,10 +21,8 @@ define i64 @hoge(i1 %c) {
 ; CHECK:       bb1:
 ; CHECK-NEXT:    br i1 [[C]], label [[BB1]], label [[BB4_LOOPEXIT:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       bb4.loopexit:
-; CHECK-NEXT:    [[TMP5_PH:%.*]] = phi i32 [ 8, [[BB1]] ]
 ; CHECK-NEXT:    br label [[BB4]]
 ; CHECK:       bb4:
-; CHECK-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, [[BB1_PEEL]] ], [ 8, [[BB2_PEEL]] ], [ [[TMP5_PH]], [[BB4_LOOPEXIT]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 (...) @llvm.experimental.deoptimize.i64(i32 10) [ "deopt"() ]
 ; CHECK-NEXT:    ret i64 [[TMP6]]
 ;
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-header-exiting-with-phis-multiple-exiting-blocks.ll b/llvm/test/Transforms/LoopUnroll/unroll-header-exiting-with-phis-multiple-exiting-blocks.ll
index 91f2e6a1f25ad..c1680dd274d00 100644
--- a/llvm/test/Transforms/LoopUnroll/unroll-header-exiting-with-phis-multiple-exiting-blocks.ll
+++ b/llvm/test/Transforms/LoopUnroll/unroll-header-exiting-with-phis-multiple-exiting-blocks.ll
@@ -8,52 +8,25 @@ define i16 @full_unroll_multiple_exiting_blocks(ptr %A, i16 %x, i16 %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[LV:%.*]] = load i16, ptr [[A:%.*]], align 2
-; CHECK-NEXT:    [[RES_NEXT:%.*]] = add i16 123, [[LV]]
-; CHECK-NEXT:    br label [[EXITING_1:%.*]]
-; CHECK:       exiting.1:
-; CHECK-NEXT:    [[EC_1:%.*]] = icmp eq i16 [[LV]], [[X:%.*]]
-; CHECK-NEXT:    br i1 [[EC_1]], label [[EXIT:%.*]], label [[EXITING_2:%.*]]
-; CHECK:       exiting.2:
-; CHECK-NEXT:    [[EC_2:%.*]] = icmp eq i16 [[LV]], [[Y:%.*]]
-; CHECK-NEXT:    br i1 [[EC_2]], label [[EXIT]], label [[LATCH:%.*]]
-; CHECK:       latch:
-; CHECK-NEXT:    [[PTR_1:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 1
+; CHECK-NEXT:    [[RES_NEXT:%.*]] = phi i16 [ 123, [[ENTRY:%.*]] ], [ [[RES_NEXT_1:%.*]], [[LATCH_3:%.*]] ]
+; CHECK-NEXT:    [[I_0:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INC9:%.*]], [[LATCH_3]] ]
+; CHECK-NEXT:    [[PTR_1:%.*]] = getelementptr inbounds i16, ptr [[A:%.*]], i64 [[I_0]]
 ; CHECK-NEXT:    [[LV_1:%.*]] = load i16, ptr [[PTR_1]], align 2
-; CHECK-NEXT:    [[RES_NEXT_1:%.*]] = add i16 [[RES_NEXT]], [[LV_1]]
-; CHECK-NEXT:    br label [[EXITING_1_1:%.*]]
-; CHECK:       exiting.1.1:
-; CHECK-NEXT:    [[EC_1_1:%.*]] = icmp eq i16 [[LV_1]], [[X]]
-; CHECK-NEXT:    br i1 [[EC_1_1]], label [[EXIT]], label [[EXITING_2_1:%.*]]
-; CHECK:       exiting.2.1:
-; CHECK-NEXT:    [[EC_2_1:%.*]] = icmp eq i16 [[LV_1]], [[Y]]
-; CHECK-NEXT:    br i1 [[EC_2_1]], label [[EXIT]], label [[LATCH_1:%.*]]
-; CHECK:       latch.1:
-; CHECK-NEXT:    [[PTR_2:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 2
-; CHECK-NEXT:    [[LV_2:%.*]] = load i16, ptr [[PTR_2]], align 2
-; CHECK-NEXT:    [[RES_NEXT_2:%.*]] = add i16 [[RES_NEXT_1]], [[LV_2]]
-; CHECK-NEXT:    br label [[EXITING_1_2:%.*]]
-; CHECK:       exiting.1.2:
-; CHECK-NEXT:    [[EC_1_2:%.*]] = icmp eq i16 [[LV_2]], [[X]]
-; CHECK-NEXT:    br i1 [[EC_1_2]], label [[EXIT]], label [[EXITING_2_2:%.*]]
-; CHECK:       exiting.2.2:
-; CHECK-NEXT:    [[EC_2_2:%.*]] = icmp eq i16 [[LV_2]], [[Y]]
-; CHECK-NEXT:    br i1 [[EC_2_2]], label [[EXIT]], label [[LATCH_2:%.*]]
-; CHECK:       latch.2:
-; CHECK-NEXT:    [[PTR_3:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 3
-; CHECK-NEXT:    [[LV_3:%.*]] = load i16, ptr [[PTR_3]], align 2
-; CHECK-NEXT:    [[RES_NEXT_3:%.*]] = add i16 [[RES_NEXT_2]], [[LV_3]]
-; CHECK-NEXT:    br i1 false, label [[EXITING_1_3:%.*]], label [[EXIT]]
-; CHECK:       exiting.1.3:
-; CHECK-NEXT:    [[EC_1_3:%.*]] = icmp eq i16 [[LV_3]], [[X]]
-; CHECK-NEXT:    br i1 [[EC_1_3]], label [[EXIT]], label [[EXITING_2_3:%.*]]
-; CHECK:       exiting.2.3:
-; CHECK-NEXT:    [[EC_2_3:%.*]] = icmp eq i16 [[LV_3]], [[Y]]
-; CHECK-NEXT:    br i1 [[EC_2_3]], label [[EXIT]], label [[LATCH_3:%.*]]
-; CHECK:       latch.3:
-; CHECK-NEXT:    unreachable
+; CHECK-NEXT:    [[RES_NEXT_1]] = add i16 [[RES_NEXT]], [[LV_1]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[I_0]], 3
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = xor i1 [[CMP]], true
+; CHECK-NEXT:    [[EC_1_1:%.*]] = icmp eq i16 [[LV_1]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = select i1 [[CMP]], i16 0, i16 [[RES_NEXT_1]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP_NOT]], i1 true, i1 [[EC_1_1]]
+; CHECK-NEXT:    [[EC_2_1:%.*]] = icmp eq i16 [[LV_1]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[OR_COND]], i16 [[TMP0]], i16 1
+; CHECK-NEXT:    [[EC_2_3:%.*]] = select i1 [[OR_COND]], i1 true, i1 [[EC_2_1]]
+; CHECK-NEXT:    br i1 [[EC_2_3]], label [[EXIT:%.*]], label [[LATCH_3]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[INC9]] = add i64 [[I_0]], 1
+; CHECK-NEXT:    br label [[HEADER]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[RES_LCSSA:%.*]] = phi i16 [ 0, [[EXITING_1]] ], [ 1, [[EXITING_2]] ], [ 0, [[EXITING_1_1]] ], [ 1, [[EXITING_2_1]] ], [ 0, [[EXITING_1_2]] ], [ 1, [[EXITING_2_2]] ], [ [[RES_NEXT_3]], [[LATCH_2]] ], [ 0, [[EXITING_1_3]] ], [ 1, [[EXITING_2_3]] ]
+; CHECK-NEXT:    [[RES_LCSSA:%.*]] = phi i16 [ [[TMP1]], [[HEADER]] ]
 ; CHECK-NEXT:    ret i16 [[RES_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
index de455c81d363e..0b821b4359fcd 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
@@ -323,7 +323,9 @@ loop.end:
 ; We don't currently support multiple uncountable early exits.
 define i64 @multiple_uncountable_exits() {
 ; CHECK-LABEL: LV: Checking a loop in 'multiple_uncountable_exits'
-; CHECK:       LV: Not vectorizing: Loop has too many uncountable exits.
+; CHECK:       LV: Found an early exit loop with symbolic max backedge taken count: 63
+; CHECK-NEXT:  LV: We can vectorize this loop!
+; CHECK-NOT:   LV: Not vectorizing:
 entry:
   %p1 = alloca [1024 x i8]
   %p2 = alloca [1024 x i8]
diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
index 5ff43dcf42bcf..e2b70eee5bb3b 100644
--- a/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
+++ b/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
@@ -1057,46 +1057,19 @@ define ptr @same_exit_block_post_inc_use1_ivptr() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[P1:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
-; CHECK-NEXT:    [[PTREND:%.*]] = getelementptr i8, ptr [[P1]], i64 1024
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[P1]], i64 1024
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[P1]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], splat (i8 72)
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP16:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP15]])
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT:    [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]]
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
-; CHECK:       middle.split:
-; CHECK-NEXT:    br i1 [[TMP16]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
-; CHECK:       vector.early.exit:
-; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP15]], i1 true)
-; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
-; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], 1
-; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP9]]
-; CHECK-NEXT:    br label [[LOOP_END]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP0]], [[MIDDLE_BLOCK]] ], [ [[P1]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[PTR:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[PTR:%.*]] = phi ptr [ [[P1]], [[ENTRY:%.*]] ], [ [[PTR_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[PTR]], align 1
 ; CHECK-NEXT:    [[PTR_NEXT]] = getelementptr inbounds i8, ptr [[PTR]], i64 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], 72
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
-; CHECK:       loop.inc:
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne ptr [[PTR_NEXT]], [[PTREND]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne ptr [[PTR_NEXT]], [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[CMP3]], ptr [[TMP0]], ptr [[PTR_NEXT]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[CMP3]], i1 [[EXITCOND]], i1 false
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[LOOP]], label [[LOOP_END:%.*]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi ptr [ [[PTR_NEXT]], [[LOOP]] ], [ [[PTREND]], [[LOOP_INC]] ], [ [[PTREND]], [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi ptr [ [[TMP1]], [[LOOP]] ]
 ; CHECK-NEXT:    ret ptr [[RETVAL]]
 ;
 entry:
@@ -1129,51 +1102,21 @@ define i64 @same_exit_block_post_inc_use2() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
-; CHECK-NEXT:    [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP17]])
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
-; CHECK-NEXT:    [[TMP20:%.*]] = or i1 [[TMP18]], [[TMP19]]
-; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
-; CHECK:       middle.split:
-; CHECK-NEXT:    br i1 [[TMP18]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
-; CHECK:       vector.early.exit:
-; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP17]], i1 true)
-; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX1]], [[FIRST_ACTIVE_LANE]]
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 1
-; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP11]]
-; CHECK-NEXT:    br label [[LOOP_END]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
-; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK-NEXT:    [[TMP0:%.*]] = select i1 [[CMP3]], i64 [[INDEX]], i64 [[INDEX_NEXT]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP3]], [[EXITCOND]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[LOOP]], label [[LOOP_END:%.*]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX_NEXT]], [[LOOP]] ], [ [[INDEX]], [[LOOP_INC]] ], [ 66, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[TMP0]], [[LOOP]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -1227,7 +1170,7 @@ define i64 @diff_exit_block_pre_inc_use1() {
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -1251,7 +1194,7 @@ define i64 @diff_exit_block_pre_inc_use1() {
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP29:![0-9]+]]
 ; CHECK:       loop.early.exit:
 ; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL1]]
@@ -1314,7 +1257,7 @@ define i64 @diff_exit_block_pre_inc_use2() {
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -1335,7 +1278,7 @@ define i64 @diff_exit_block_pre_inc_use2() {
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP31:![0-9]+]]
 ; CHECK:       loop.early.exit:
 ; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ 67, [[LOOP]] ], [ 67, [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL1]]
@@ -1398,7 +1341,7 @@ define i64 @diff_exit_block_pre_inc_use3() {
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 64
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -1422,7 +1365,7 @@ define i64 @diff_exit_block_pre_inc_use3() {
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP37:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP33:![0-9]+]]
 ; CHECK:       loop.early.exit:
 ; CHECK-NEXT:    [[INDEX_LCSSA:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[INDEX_LCSSA]]
@@ -1483,7 +1426,7 @@ define i64 @diff_exit_block_post_inc_use1() {
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]])
 ; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
 ; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[TMP14]], [[TMP15]]
-; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP14]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -1507,7 +1450,7 @@ define i64 @diff_exit_block_post_inc_use1() {
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP39:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP35:![0-9]+]]
 ; CHECK:       loop.early.exit:
 ; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL1]]
@@ -1570,7 +1513,7 @@ define i64 @diff_exit_block_post_inc_use2() {
 ; CHECK-NEXT:    [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP17]])
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
 ; CHECK-NEXT:    [[TMP20:%.*]] = or i1 [[TMP18]], [[TMP19]]
-; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -1595,7 +1538,7 @@ define i64 @diff_exit_block_post_inc_use2() {
 ; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_EARLY_EXIT]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP41:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP37:![0-9]+]]
 ; CHECK:       loop.early.exit:
 ; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ [[INDEX_NEXT]], [[LOOP]] ], [ [[TMP21]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL1]]
@@ -1659,7 +1602,7 @@ define i64 @diff_exit_block_post_inc_use3(i64 %start) {
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP19]])
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 64
 ; CHECK-NEXT:    [[TMP22:%.*]] = or i1 [[TMP20]], [[TMP21]]
-; CHECK-NEXT:    br i1 [[TMP22]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP22]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP20]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -1688,7 +1631,7 @@ define i64 @diff_exit_block_post_inc_use3(i64 %start) {
 ; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_EARLY_EXIT]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP43:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP39:![0-9]+]]
 ; CHECK:       loop.early.exit:
 ; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ [[INDEX2_NEXT]], [[LOOP]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL1]]
@@ -1751,7 +1694,7 @@ define i64 @loop_contains_safe_call() {
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
 ; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -1774,7 +1717,7 @@ define i64 @loop_contains_safe_call() {
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP45:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP41:![0-9]+]]
 ; CHECK:       loop.end:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
@@ -1827,7 +1770,7 @@ define i64 @loop_contains_safe_div() {
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
 ; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -1850,7 +1793,7 @@ define i64 @loop_contains_safe_div() {
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP47:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP43:![0-9]+]]
 ; CHECK:       loop.end:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
@@ -1904,7 +1847,7 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[WIDE_LOAD2]], i32 3
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
@@ -1929,7 +1872,7 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(
 ; CHECK-NEXT:    [[LD2:%.*]] = load i64, ptr [[ARRAYIDX2]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP49:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP45:![0-9]+]]
 ; CHECK:       loop.end:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[LD2]], [[LOOP_INC]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
@@ -1987,7 +1930,7 @@ define i64 @same_exit_block_pre_inc_use1_reverse() {
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 1020
 ; CHECK-NEXT:    [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
-; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -2011,7 +1954,7 @@ define i64 @same_exit_block_pre_inc_use1_reverse() {
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], -1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDEX_NEXT]], 0
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP_END]], label [[LOOP1]], !llvm.loop [[LOOP51:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP_END]], label [[LOOP1]], !llvm.loop [[LOOP47:![0-9]+]]
 ; CHECK:       loop.end:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP1]] ], [ 1024, [[LOOP_INC]] ], [ 1024, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
@@ -2114,7 +2057,7 @@ define i64 @same_exit_block_pre_inc_use1_deref_ptrs(ptr dereferenceable(1024) %p
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -2138,7 +2081,7 @@ define i64 @same_exit_block_pre_inc_use1_deref_ptrs(ptr dereferenceable(1024) %p
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP53:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP49:![0-9]+]]
 ; CHECK:       loop.end:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
@@ -2220,8 +2163,4 @@ attributes #0 = { "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vec)" }
 ; CHECK: [[LOOP47]] = distinct !{[[LOOP47]], [[META2]], [[META1]]}
 ; CHECK: [[LOOP48]] = distinct !{[[LOOP48]], [[META1]], [[META2]]}
 ; CHECK: [[LOOP49]] = distinct !{[[LOOP49]], [[META2]], [[META1]]}
-; CHECK: [[LOOP50]] = distinct !{[[LOOP50]], [[META1]], [[META2]]}
-; CHECK: [[LOOP51]] = distinct !{[[LOOP51]], [[META2]], [[META1]]}
-; CHECK: [[LOOP52]] = distinct !{[[LOOP52]], [[META1]], [[META2]]}
-; CHECK: [[LOOP53]] = distinct !{[[LOOP53]], [[META2]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll b/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
index 5b2a95f1b368c..d6a3c7c3936e4 100644
--- a/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
@@ -65,24 +65,57 @@ define i64 @multiple_uncountable_exits() {
 ; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
 ; CHECK-NEXT:    br label [[SEARCH1:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[SEARCH1]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[SEARCH1]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult <4 x i8> [[WIDE_LOAD]], splat (i8 34)
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> [[VEC_IND]], <4 x i64> splat (i64 100)
+; CHECK-NEXT:    [[TMP7:%.*]] = or <4 x i1> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_SPLIT:%.*]], label [[SEARCH1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP8]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP7]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[TMP6]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_END]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[SEARCH2:%.*]]
 ; CHECK:       search1:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT:    br i1 [[CMP1]], label [[LOOP_END:%.*]], label [[SEARCH2:%.*]]
-; CHECK:       search2:
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 [[LD1]], 34
-; CHECK-NEXT:    br i1 [[CMP2]], label [[LOOP_END]], label [[LOOP_INC]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select i1 [[CMP1]], i64 [[INDEX]], i64 100
+; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[LOOP_END]], label [[LOOP_INC]]
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[SEARCH1]], label [[LOOP_END]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[SEARCH2]], label [[LOOP_END]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[SEARCH1]] ], [ 100, [[SEARCH2]] ], [ 43, [[LOOP_INC]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ 43, [[LOOP_INC]] ], [ [[TMP11]], [[SEARCH2]] ], [ 43, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -179,7 +212,7 @@ define i64 @loop_contains_unsafe_call() {
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i32, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT:    [[BAD_CALL:%.*]] = call i32 @foo(i32 [[LD1]]) #[[ATTR1:[0-9]+]]
+; CHECK-NEXT:    [[BAD_CALL:%.*]] = call i32 @foo(i32 [[LD1]]) #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[BAD_CALL]], 34
 ; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
 ; CHECK:       loop.inc:
@@ -492,3 +525,9 @@ declare i32 @foo(i32) readonly
 declare <vscale x 4 x i32> @foo_vec(<vscale x 4 x i32>)
 
 attributes #0 = { "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vec)" }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/inject-invariant-conditions.ll b/llvm/test/Transforms/SimpleLoopUnswitch/inject-invariant-conditions.ll
index 536e0c6a0e74a..c856efe3a2284 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/inject-invariant-conditions.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/inject-invariant-conditions.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
-; RUN: opt < %s -S -simple-loop-unswitch-inject-invariant-conditions=true -passes="loop(simple-loop-unswitch<nontrivial>),simplifycfg" | FileCheck %s
-; RUN: opt < %s -S -simple-loop-unswitch-inject-invariant-conditions=true -passes="loop-mssa(simple-loop-unswitch<nontrivial>),simplifycfg" -verify-memoryssa | FileCheck %s
+; RUN: opt < %s -S -simple-loop-unswitch-inject-invariant-conditions=true -simplifycfg-branch-fold-threshold=0 -passes="loop(simple-loop-unswitch<nontrivial>),simplifycfg" | FileCheck %s
+; RUN: opt < %s -S -simple-loop-unswitch-inject-invariant-conditions=true -simplifycfg-branch-fold-threshold=0 -passes="loop-mssa(simple-loop-unswitch<nontrivial>),simplifycfg" -verify-memoryssa | FileCheck %s
 
 define i32 @test_01(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noundef %arr, ptr noundef %x_p) {
 ; CHECK-LABEL: @test_01(
diff --git a/llvm/test/Transforms/SimplifyCFG/2008-07-13-InfLoopMiscompile.ll b/llvm/test/Transforms/SimplifyCFG/2008-07-13-InfLoopMiscompile.ll
index 2e9e7b19c73e2..a22e28685a124 100644
--- a/llvm/test/Transforms/SimplifyCFG/2008-07-13-InfLoopMiscompile.ll
+++ b/llvm/test/Transforms/SimplifyCFG/2008-07-13-InfLoopMiscompile.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s
+; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -simplifycfg-branch-fold-threshold=0 -S | FileCheck %s
 ; PR2540
 ; Outval should end up with a select from 0/2, not all constants.
 
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/SpeculativeExec.ll b/llvm/test/Transforms/SimplifyCFG/X86/SpeculativeExec.ll
index 2de1c72c4007a..79248eacb401a 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/SpeculativeExec.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/SpeculativeExec.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -phi-node-folding-threshold=2 -S | FileCheck %s
+; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -phi-node-folding-threshold=2 -simplifycfg-branch-fold-threshold=0 -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
index 95bf296af9b0c..bb17a30de32e1 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
@@ -1451,9 +1451,10 @@ define i32 @no_reuse_cmp2(i32 %x, i32 %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[EC:%.*]] = icmp ne i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[EC]], i32 0, i32 100
+; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[EC]], i1 [[TMP0]], i1 false
 ; CHECK-NEXT:    [[SWITCH_OFFSET:%.*]] = add nsw i32 [[X]], 10
-; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP0]], i32 [[SWITCH_OFFSET]], i32 0
-; CHECK-NEXT:    [[R_0:%.*]] = select i1 [[EC]], i32 [[SPEC_SELECT]], i32 100
+; CHECK-NEXT:    [[R_0:%.*]] = select i1 [[OR_COND]], i32 [[SWITCH_OFFSET]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[R_0]], 0
 ; CHECK-NEXT:    [[DOTR_0:%.*]] = select i1 [[CMP]], i32 100, i32 [[R_0]]
 ; CHECK-NEXT:    ret i32 [[DOTR_0]]
diff --git a/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest-phis.ll b/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest-phis.ll
index 150bdfcff92a1..7f7274fa53430 100644
--- a/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest-phis.ll
+++ b/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest-phis.ll
@@ -12,18 +12,16 @@ define void @incompatible_ivs_of_single_phi.falsedest.falsedest(i8 %v0, i8 %v1,
 ; ALL-LABEL: @incompatible_ivs_of_single_phi.falsedest.falsedest(
 ; ALL-NEXT:  pred:
 ; ALL-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
-; ALL-NEXT:    br i1 [[C0]], label [[DISPATCH:%.*]], label [[FINAL_RIGHT:%.*]]
-; ALL:       dispatch:
 ; ALL-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; ALL-NEXT:    br i1 [[C1]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT]]
+; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = select i1 [[C0]], i8 [[IV_OF_FINAL_RIGHT_FROM_DISPATCH:%.*]], i8 [[IV_OF_FINAL_RIGHT_FROM_PRED:%.*]]
+; ALL-NEXT:    [[OR_COND:%.*]] = select i1 [[C0]], i1 [[C1]], i1 false
+; ALL-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; ALL:       common.ret:
 ; ALL-NEXT:    ret void
 ; ALL:       final_left:
-; ALL-NEXT:    [[FINAL_LEFT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_LEFT_FROM_DISPATCH:%.*]], [[DISPATCH]] ]
-; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI]])
+; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI:%.*]])
 ; ALL-NEXT:    br label [[COMMON_RET:%.*]]
 ; ALL:       final_right:
-; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_RIGHT_FROM_PRED:%.*]], [[PRED:%.*]] ], [ [[IV_OF_FINAL_RIGHT_FROM_DISPATCH:%.*]], [[DISPATCH]] ]
 ; ALL-NEXT:    call void @sideeffect1(i8 [[FINAL_RIGHT_PHI]])
 ; ALL-NEXT:    br label [[COMMON_RET]]
 ;
@@ -48,18 +46,16 @@ define void @incompatible_ivs_of_single_phi.falsedest.falsedest.extrause(i8 %v0,
 ; ALL-NEXT:  pred:
 ; ALL-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
 ; ALL-NEXT:    call void @use1(i1 [[C0]])
-; ALL-NEXT:    br i1 [[C0]], label [[DISPATCH:%.*]], label [[FINAL_RIGHT:%.*]]
-; ALL:       dispatch:
 ; ALL-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; ALL-NEXT:    br i1 [[C1]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT]]
+; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = select i1 [[C0]], i8 [[IV_OF_FINAL_RIGHT_FROM_DISPATCH:%.*]], i8 [[IV_OF_FINAL_RIGHT_FROM_PRED:%.*]]
+; ALL-NEXT:    [[OR_COND:%.*]] = select i1 [[C0]], i1 [[C1]], i1 false
+; ALL-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; ALL:       common.ret:
 ; ALL-NEXT:    ret void
 ; ALL:       final_left:
-; ALL-NEXT:    [[FINAL_LEFT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_LEFT_FROM_DISPATCH:%.*]], [[DISPATCH]] ]
-; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI]])
+; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI:%.*]])
 ; ALL-NEXT:    br label [[COMMON_RET:%.*]]
 ; ALL:       final_right:
-; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_RIGHT_FROM_PRED:%.*]], [[PRED:%.*]] ], [ [[IV_OF_FINAL_RIGHT_FROM_DISPATCH:%.*]], [[DISPATCH]] ]
 ; ALL-NEXT:    call void @sideeffect1(i8 [[FINAL_RIGHT_PHI]])
 ; ALL-NEXT:    br label [[COMMON_RET]]
 ;
@@ -84,18 +80,17 @@ define void @incompatible_ivs_of_single_phi.falsedest.truedest(i8 %v0, i8 %v1, i
 ; ALL-LABEL: @incompatible_ivs_of_single_phi.falsedest.truedest(
 ; ALL-NEXT:  pred:
 ; ALL-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
-; ALL-NEXT:    br i1 [[C0]], label [[DISPATCH:%.*]], label [[FINAL_RIGHT:%.*]]
-; ALL:       dispatch:
+; ALL-NEXT:    [[C0_NOT:%.*]] = xor i1 [[C0]], true
 ; ALL-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; ALL-NEXT:    br i1 [[C1]], label [[FINAL_RIGHT]], label [[FINAL_LEFT:%.*]]
+; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = select i1 [[C0]], i8 [[IV_OF_FINAL_RIGHT_FROM_DISPATCH:%.*]], i8 [[IV_OF_FINAL_RIGHT_FROM_PRED:%.*]]
+; ALL-NEXT:    [[OR_COND:%.*]] = select i1 [[C0_NOT]], i1 true, i1 [[C1]]
+; ALL-NEXT:    br i1 [[OR_COND]], label [[FINAL_RIGHT:%.*]], label [[FINAL_LEFT:%.*]]
 ; ALL:       common.ret:
 ; ALL-NEXT:    ret void
 ; ALL:       final_left:
-; ALL-NEXT:    [[FINAL_LEFT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_LEFT_FROM_DISPATCH:%.*]], [[DISPATCH]] ]
-; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI]])
+; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI:%.*]])
 ; ALL-NEXT:    br label [[COMMON_RET:%.*]]
 ; ALL:       final_right:
-; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_RIGHT_FROM_PRED:%.*]], [[PRED:%.*]] ], [ [[IV_OF_FINAL_RIGHT_FROM_DISPATCH:%.*]], [[DISPATCH]] ]
 ; ALL-NEXT:    call void @sideeffect1(i8 [[FINAL_RIGHT_PHI]])
 ; ALL-NEXT:    br label [[COMMON_RET]]
 ;
@@ -156,18 +151,17 @@ define void @incompatible_ivs_of_single_phi.truedest.falsedest(i8 %v0, i8 %v1, i
 ; ALL-LABEL: @incompatible_ivs_of_single_phi.truedest.falsedest(
 ; ALL-NEXT:  pred:
 ; ALL-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
-; ALL-NEXT:    br i1 [[C0]], label [[FINAL_RIGHT:%.*]], label [[DISPATCH:%.*]]
-; ALL:       dispatch:
+; ALL-NEXT:    [[C0_NOT:%.*]] = xor i1 [[C0]], true
 ; ALL-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; ALL-NEXT:    br i1 [[C1]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT]]
+; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = select i1 [[C0]], i8 [[IV_OF_FINAL_RIGHT_FROM_PRED:%.*]], i8 [[IV_OF_FINAL_RIGHT_FROM_DISPATCH:%.*]]
+; ALL-NEXT:    [[OR_COND:%.*]] = select i1 [[C0_NOT]], i1 [[C1]], i1 false
+; ALL-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; ALL:       common.ret:
 ; ALL-NEXT:    ret void
 ; ALL:       final_left:
-; ALL-NEXT:    [[FINAL_LEFT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_LEFT_FROM_DISPATCH:%.*]], [[DISPATCH]] ]
-; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI]])
+; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI:%.*]])
 ; ALL-NEXT:    br label [[COMMON_RET:%.*]]
 ; ALL:       final_right:
-; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_RIGHT_FROM_PRED:%.*]], [[PRED:%.*]] ], [ [[IV_OF_FINAL_RIGHT_FROM_DISPATCH:%.*]], [[DISPATCH]] ]
 ; ALL-NEXT:    call void @sideeffect1(i8 [[FINAL_RIGHT_PHI]])
 ; ALL-NEXT:    br label [[COMMON_RET]]
 ;
@@ -228,18 +222,16 @@ define void @incompatible_ivs_of_single_phi.truedest.truedest(i8 %v0, i8 %v1, i8
 ; ALL-LABEL: @incompatible_ivs_of_single_phi.truedest.truedest(
 ; ALL-NEXT:  pred:
 ; ALL-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
-; ALL-NEXT:    br i1 [[C0]], label [[FINAL_RIGHT:%.*]], label [[DISPATCH:%.*]]
-; ALL:       dispatch:
 ; ALL-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; ALL-NEXT:    br i1 [[C1]], label [[FINAL_RIGHT]], label [[FINAL_LEFT:%.*]]
+; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = select i1 [[C0]], i8 [[IV_OF_FINAL_RIGHT_FROM_PRED:%.*]], i8 [[IV_OF_FINAL_RIGHT_FROM_DISPATCH:%.*]]
+; ALL-NEXT:    [[OR_COND:%.*]] = select i1 [[C0]], i1 true, i1 [[C1]]
+; ALL-NEXT:    br i1 [[OR_COND]], label [[FINAL_RIGHT:%.*]], label [[FINAL_LEFT:%.*]]
 ; ALL:       common.ret:
 ; ALL-NEXT:    ret void
 ; ALL:       final_left:
-; ALL-NEXT:    [[FINAL_LEFT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_LEFT_FROM_DISPATCH:%.*]], [[DISPATCH]] ]
-; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI]])
+; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI:%.*]])
 ; ALL-NEXT:    br label [[COMMON_RET:%.*]]
 ; ALL:       final_right:
-; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_RIGHT_FROM_PRED:%.*]], [[PRED:%.*]] ], [ [[IV_OF_FINAL_RIGHT_FROM_DISPATCH:%.*]], [[DISPATCH]] ]
 ; ALL-NEXT:    call void @sideeffect1(i8 [[FINAL_RIGHT_PHI]])
 ; ALL-NEXT:    br label [[COMMON_RET]]
 ;
@@ -264,18 +256,16 @@ define void @incompatible_ivs_of_single_phi.truedest.truedest.extrause(i8 %v0, i
 ; ALL-NEXT:  pred:
 ; ALL-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
 ; ALL-NEXT:    call void @use1(i1 [[C0]])
-; ALL-NEXT:    br i1 [[C0]], label [[FINAL_RIGHT:%.*]], label [[DISPATCH:%.*]]
-; ALL:       dispatch:
 ; ALL-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; ALL-NEXT:    br i1 [[C1]], label [[FINAL_RIGHT]], label [[FINAL_LEFT:%.*]]
+; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = select i1 [[C0]], i8 [[IV_OF_FINAL_RIGHT_FROM_PRED:%.*]], i8 [[IV_OF_FINAL_RIGHT_FROM_DISPATCH:%.*]]
+; ALL-NEXT:    [[OR_COND:%.*]] = select i1 [[C0]], i1 true, i1 [[C1]]
+; ALL-NEXT:    br i1 [[OR_COND]], label [[FINAL_RIGHT:%.*]], label [[FINAL_LEFT:%.*]]
 ; ALL:       common.ret:
 ; ALL-NEXT:    ret void
 ; ALL:       final_left:
-; ALL-NEXT:    [[FINAL_LEFT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_LEFT_FROM_DISPATCH:%.*]], [[DISPATCH]] ]
-; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI]])
+; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI:%.*]])
 ; ALL-NEXT:    br label [[COMMON_RET:%.*]]
 ; ALL:       final_right:
-; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_RIGHT_FROM_PRED:%.*]], [[PRED:%.*]] ], [ [[IV_OF_FINAL_RIGHT_FROM_DISPATCH:%.*]], [[DISPATCH]] ]
 ; ALL-NEXT:    call void @sideeffect1(i8 [[FINAL_RIGHT_PHI]])
 ; ALL-NEXT:    br label [[COMMON_RET]]
 ;
@@ -303,19 +293,17 @@ define void @incompatible_ivs_of_single_phi.insertpos(i8 %v0, i8 %v1, i8 %iv.of.
 ; ALL-NEXT:  pred:
 ; ALL-NEXT:    [[IV_OF_FINAL_RIGHT_FROM_PRED:%.*]] = load i8, ptr [[SRC0:%.*]], align 1
 ; ALL-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
-; ALL-NEXT:    br i1 [[C0]], label [[DISPATCH:%.*]], label [[FINAL_RIGHT:%.*]]
-; ALL:       dispatch:
 ; ALL-NEXT:    [[IV_OF_FINAL_RIGHT_FROM_DISPATCH:%.*]] = load i8, ptr [[SRC1:%.*]], align 1
 ; ALL-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; ALL-NEXT:    br i1 [[C1]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT]]
+; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = select i1 [[C0]], i8 [[IV_OF_FINAL_RIGHT_FROM_DISPATCH]], i8 [[IV_OF_FINAL_RIGHT_FROM_PRED]]
+; ALL-NEXT:    [[OR_COND:%.*]] = select i1 [[C0]], i1 [[C1]], i1 false
+; ALL-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; ALL:       common.ret:
 ; ALL-NEXT:    ret void
 ; ALL:       final_left:
-; ALL-NEXT:    [[FINAL_LEFT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_LEFT_FROM_DISPATCH:%.*]], [[DISPATCH]] ]
-; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI]])
+; ALL-NEXT:    call void @sideeffect0(i8 [[FINAL_LEFT_PHI:%.*]])
 ; ALL-NEXT:    br label [[COMMON_RET:%.*]]
 ; ALL:       final_right:
-; ALL-NEXT:    [[FINAL_RIGHT_PHI:%.*]] = phi i8 [ [[IV_OF_FINAL_RIGHT_FROM_PRED]], [[PRED:%.*]] ], [ [[IV_OF_FINAL_RIGHT_FROM_DISPATCH]], [[DISPATCH]] ]
 ; ALL-NEXT:    call void @sideeffect1(i8 [[FINAL_RIGHT_PHI]])
 ; ALL-NEXT:    br label [[COMMON_RET]]
 ;
@@ -459,10 +447,10 @@ define float @D139275_c4001580(float %val) {
 ; ALL-LABEL: @D139275_c4001580(
 ; ALL-NEXT:  entry:
 ; ALL-NEXT:    [[CMP:%.*]] = fcmp ugt float [[VAL:%.*]], 0.000000e+00
-; ALL-NEXT:    br i1 [[CMP]], label [[IF_END:%.*]], label [[RETURN:%.*]]
-; ALL:       if.end:
 ; ALL-NEXT:    [[CMP1:%.*]] = fcmp ult float [[VAL]], 1.000000e+00
-; ALL-NEXT:    br i1 [[CMP1]], label [[IF_END3:%.*]], label [[RETURN]]
+; ALL-NEXT:    [[TMP0:%.*]] = select i1 [[CMP]], float 0x3FB99999A0000000, float 0.000000e+00
+; ALL-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP]], [[CMP1]]
+; ALL-NEXT:    br i1 [[OR_COND]], label [[IF_END3:%.*]], label [[RETURN:%.*]]
 ; ALL:       if.end3:
 ; ALL-NEXT:    [[CMP4:%.*]] = fcmp olt float [[VAL]], 0x3FC99999A0000000
 ; ALL-NEXT:    br i1 [[CMP4]], label [[RETURN]], label [[IF_END6:%.*]]
@@ -470,7 +458,7 @@ define float @D139275_c4001580(float %val) {
 ; ALL-NEXT:    [[SUB:%.*]] = fadd float [[VAL]], 0xBFB99999A0000000
 ; ALL-NEXT:    br label [[RETURN]]
 ; ALL:       return:
-; ALL-NEXT:    [[RETVAL_0:%.*]] = phi float [ [[SUB]], [[IF_END6]] ], [ 0.000000e+00, [[ENTRY:%.*]] ], [ 0x3FB99999A0000000, [[IF_END]] ], [ 0.000000e+00, [[IF_END3]] ]
+; ALL-NEXT:    [[RETVAL_0:%.*]] = phi float [ [[SUB]], [[IF_END6]] ], [ 0.000000e+00, [[IF_END3]] ], [ [[TMP0]], [[ENTRY:%.*]] ]
 ; ALL-NEXT:    ret float [[RETVAL_0]]
 ;
 entry:
@@ -494,6 +482,51 @@ return:
   ret float %retval.0
 }
 
+define i1 @_Z7compareRK1SS1_(ptr %a, ptr %b) {
+; ALL-LABEL: @_Z7compareRK1SS1_(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A:%.*]], align 4
+; ALL-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B:%.*]], align 4
+; ALL-NEXT:    [[CMP_I:%.*]] = icmp slt i32 [[TMP0]], [[TMP1]]
+; ALL-NEXT:    [[CMP_I_NOT:%.*]] = xor i1 [[CMP_I]], true
+; ALL-NEXT:    [[CMP_I19:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
+; ALL-NEXT:    [[TMP2:%.*]] = select i1 [[CMP_I]], i1 true, i1 false
+; ALL-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP_I_NOT]], [[CMP_I19]]
+; ALL-NEXT:    br i1 [[OR_COND]], label [[LAND_RHS:%.*]], label [[LOR_END:%.*]]
+; ALL:       land.rhs:
+; ALL-NEXT:    [[Y:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 4
+; ALL-NEXT:    [[TMP3:%.*]] = load i32, ptr [[Y]], align 4
+; ALL-NEXT:    [[Y14:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 4
+; ALL-NEXT:    [[TMP4:%.*]] = load i32, ptr [[Y14]], align 4
+; ALL-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP3]], [[TMP4]]
+; ALL-NEXT:    br label [[LOR_END]]
+; ALL:       lor.end:
+; ALL-NEXT:    [[TMP5:%.*]] = phi i1 [ [[CMP]], [[LAND_RHS]] ], [ [[TMP2]], [[ENTRY:%.*]] ]
+; ALL-NEXT:    ret i1 [[TMP5]]
+;
+entry:
+  %0 = load i32, ptr %a, align 4
+  %1 = load i32, ptr %b, align 4
+  %cmp.i = icmp slt i32 %0, %1
+  br i1 %cmp.i, label %lor.end, label %lor.rhs
+
+lor.rhs:                                          ; preds = %entry
+  %cmp.i19 = icmp eq i32 %0, %1
+  br i1 %cmp.i19, label %land.rhs, label %lor.end
+
+land.rhs:                                         ; preds = %lor.rhs
+  %y = getelementptr inbounds nuw i8, ptr %a, i64 4
+  %2 = load i32, ptr %y, align 4
+  %y14 = getelementptr inbounds nuw i8, ptr %b, i64 4
+  %3 = load i32, ptr %y14, align 4
+  %cmp = icmp slt i32 %2, %3
+  br label %lor.end
+
+lor.end:                                          ; preds = %lor.rhs, %land.rhs, %entry
+  %4 = phi i1 [ true, %entry ], [ false, %lor.rhs ], [ %cmp, %land.rhs ]
+  ret i1 %4
+}
+
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; CHEAP: {{.*}}
 ; COSTLY: {{.*}}
diff --git a/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll b/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll
index 7b88ec338cf5e..dd18d5121c642 100644
--- a/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll
+++ b/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll
@@ -479,21 +479,21 @@ define void @two_preds_with_extra_op_liveout(i8 %v0, i8 %v1, i8 %v2, i8 %v3) {
 ; CHECK-NEXT:    br i1 [[C0]], label [[PRED0:%.*]], label [[PRED1:%.*]]
 ; CHECK:       pred0:
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; CHECK-NEXT:    br i1 [[C1]], label [[FINAL_LEFT:%.*]], label [[DISPATCH:%.*]]
+; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2:%.*]]
+; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3_ADJ_OLD]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = select i1 [[C1]], i8 0, i8 [[V3_ADJ_OLD]]
+; CHECK-NEXT:    [[OR_COND1:%.*]] = select i1 [[C1]], i1 true, i1 [[C3_OLD]]
+; CHECK-NEXT:    br i1 [[OR_COND1]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       pred1:
-; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
+; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2]], 0
 ; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3_ADJ]], 0
 ; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
-; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
-; CHECK:       dispatch:
-; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2]]
-; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3_ADJ_OLD]], 0
-; CHECK-NEXT:    br i1 [[C3_OLD]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    ret void
 ; CHECK:       final_left:
-; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[V3_ADJ_OLD]], [[DISPATCH]] ], [ 0, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
+; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[V3_ADJ]], [[PRED1]] ], [ [[TMP0]], [[PRED0]] ]
 ; CHECK-NEXT:    call void @use8(i8 [[MERGE_LEFT]])
 ; CHECK-NEXT:    call void @sideeffect0()
 ; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
@@ -530,23 +530,23 @@ define void @two_preds_with_extra_op_liveout_multiuse(i8 %v0, i8 %v1, i8 %v2, i8
 ; CHECK-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
 ; CHECK-NEXT:    br i1 [[C0]], label [[PRED0:%.*]], label [[PRED1:%.*]]
 ; CHECK:       pred0:
-; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; CHECK-NEXT:    br i1 [[C1]], label [[FINAL_LEFT:%.*]], label [[DISPATCH:%.*]]
+; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3_ADJ_OLD:%.*]], 0
+; CHECK-NEXT:    br i1 [[C3_OLD]], label [[FINAL_LEFT:%.*]], label [[DISPATCH:%.*]]
 ; CHECK:       pred1:
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
-; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
+; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V3_ADJ_OLD]], [[V2]]
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3_ADJ]], 0
 ; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       dispatch:
-; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2]]
-; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3_ADJ_OLD]], 0
-; CHECK-NEXT:    br i1 [[C3_OLD]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
+; CHECK-NEXT:    [[V3_ADJ_OLD1:%.*]] = add i8 [[V3_ADJ_OLD]], [[V2]]
+; CHECK-NEXT:    [[C3_OLD1:%.*]] = icmp eq i8 [[V3_ADJ_OLD1]], 0
+; CHECK-NEXT:    br i1 [[C3_OLD1]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    ret void
 ; CHECK:       final_left:
-; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[V3_ADJ_OLD]], [[DISPATCH]] ], [ 0, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
-; CHECK-NEXT:    [[MERGE_LEFT_2:%.*]] = phi i8 [ [[V3_ADJ_OLD]], [[DISPATCH]] ], [ 42, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
+; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[V3_ADJ_OLD1]], [[DISPATCH]] ], [ 0, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
+; CHECK-NEXT:    [[MERGE_LEFT_2:%.*]] = phi i8 [ [[V3_ADJ_OLD1]], [[DISPATCH]] ], [ 42, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
 ; CHECK-NEXT:    call void @use8(i8 [[MERGE_LEFT]])
 ; CHECK-NEXT:    call void @use8(i8 [[MERGE_LEFT_2]])
 ; CHECK-NEXT:    call void @sideeffect0()
@@ -661,21 +661,21 @@ define void @two_preds_with_extra_op_externally_used_only(i8 %v0, i8 %v1, i8 %v2
 ; CHECK-NEXT:    br i1 [[C0]], label [[PRED0:%.*]], label [[PRED1:%.*]]
 ; CHECK:       pred0:
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; CHECK-NEXT:    br i1 [[C1]], label [[FINAL_LEFT:%.*]], label [[DISPATCH:%.*]]
+; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2:%.*]]
+; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3:%.*]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = select i1 [[C1]], i8 0, i8 [[V3_ADJ_OLD]]
+; CHECK-NEXT:    [[OR_COND1:%.*]] = select i1 [[C1]], i1 true, i1 [[C3_OLD]]
+; CHECK-NEXT:    br i1 [[OR_COND1]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       pred1:
-; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
+; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2]], 0
 ; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3:%.*]], 0
+; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3]], 0
 ; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
-; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
-; CHECK:       dispatch:
-; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2]]
-; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3]], 0
-; CHECK-NEXT:    br i1 [[C3_OLD]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    ret void
 ; CHECK:       final_left:
-; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[V3_ADJ_OLD]], [[DISPATCH]] ], [ 0, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
+; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[V3_ADJ]], [[PRED1]] ], [ [[TMP0]], [[PRED0]] ]
 ; CHECK-NEXT:    call void @use8(i8 [[MERGE_LEFT]])
 ; CHECK-NEXT:    call void @sideeffect0()
 ; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
@@ -712,18 +712,18 @@ define void @two_preds_with_extra_op_externally_used_only_multiuse(i8 %v0, i8 %v
 ; CHECK-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
 ; CHECK-NEXT:    br i1 [[C0]], label [[PRED0:%.*]], label [[PRED1:%.*]]
 ; CHECK:       pred0:
-; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; CHECK-NEXT:    br i1 [[C1]], label [[FINAL_LEFT:%.*]], label [[DISPATCH:%.*]]
+; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3:%.*]], 0
+; CHECK-NEXT:    br i1 [[C3_OLD]], label [[FINAL_LEFT:%.*]], label [[DISPATCH:%.*]]
 ; CHECK:       pred1:
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
-; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3:%.*]], 0
+; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V3]], [[V2]]
+; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V4:%.*]], 0
 ; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       dispatch:
-; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V1]], [[V2]]
-; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3]], 0
-; CHECK-NEXT:    br i1 [[C3_OLD]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
+; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V3]], [[V2]]
+; CHECK-NEXT:    [[C3_OLD1:%.*]] = icmp eq i8 [[V4]], 0
+; CHECK-NEXT:    br i1 [[C3_OLD1]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    ret void
 ; CHECK:       final_left:
@@ -820,21 +820,21 @@ define void @two_preds_with_extra_op_externally_used_only_after_cond(i8 %v0, i8
 ; CHECK-NEXT:    br i1 [[C0]], label [[PRED0:%.*]], label [[PRED1:%.*]]
 ; CHECK:       pred0:
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; CHECK-NEXT:    br i1 [[C1]], label [[FINAL_LEFT:%.*]], label [[DISPATCH:%.*]]
+; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3:%.*]], 0
+; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V4:%.*]], [[V5:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = select i1 [[C1]], i8 0, i8 [[V3_ADJ_OLD]]
+; CHECK-NEXT:    [[OR_COND1:%.*]] = select i1 [[C1]], i1 true, i1 [[C3_OLD]]
+; CHECK-NEXT:    br i1 [[OR_COND1]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       pred1:
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
-; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3:%.*]], 0
-; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V4:%.*]], [[V5:%.*]]
+; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3]], 0
+; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V4]], [[V5]]
 ; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[C2]], i1 [[C3]], i1 false
-; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
-; CHECK:       dispatch:
-; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3]], 0
-; CHECK-NEXT:    [[V3_ADJ_OLD:%.*]] = add i8 [[V4]], [[V5]]
-; CHECK-NEXT:    br i1 [[C3_OLD]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    ret void
 ; CHECK:       final_left:
-; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[V3_ADJ_OLD]], [[DISPATCH]] ], [ 0, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
+; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[V3_ADJ]], [[PRED1]] ], [ [[TMP0]], [[PRED0]] ]
 ; CHECK-NEXT:    call void @use8(i8 [[MERGE_LEFT]])
 ; CHECK-NEXT:    call void @sideeffect0()
 ; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
@@ -1149,13 +1149,13 @@ define i32 @test_builtin_fpclassify(float %x) {
 ; CHECK-LABEL: @test_builtin_fpclassify(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ISZERO:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
-; CHECK-NEXT:    br i1 [[ISZERO]], label [[FPCLASSIFY_END:%.*]], label [[FPCLASSIFY_NOT_ZERO:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp uno float [[X]], 0.000000e+00
+; CHECK-NEXT:    [[TMP0:%.*]] = select i1 [[ISZERO]], i32 2, i32 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[ISZERO]], [[CMP]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FPCLASSIFY_END:%.*]], label [[FPCLASSIFY_NOT_NAN:%.*]]
 ; CHECK:       fpclassify_end:
-; CHECK-NEXT:    [[FPCLASSIFY_RESULT:%.*]] = phi i32 [ 2, [[ENTRY:%.*]] ], [ 0, [[FPCLASSIFY_NOT_ZERO]] ], [ 1, [[FPCLASSIFY_NOT_NAN:%.*]] ], [ [[NORMAL_OR_SUBNORMAL:%.*]], [[FPCLASSIFY_NOT_INF:%.*]] ]
+; CHECK-NEXT:    [[FPCLASSIFY_RESULT:%.*]] = phi i32 [ 1, [[FPCLASSIFY_NOT_NAN]] ], [ [[NORMAL_OR_SUBNORMAL:%.*]], [[FPCLASSIFY_NOT_INF:%.*]] ], [ [[TMP0]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    ret i32 [[FPCLASSIFY_RESULT]]
-; CHECK:       fpclassify_not_zero:
-; CHECK-NEXT:    [[CMP:%.*]] = fcmp uno float [[X]], 0.000000e+00
-; CHECK-NEXT:    br i1 [[CMP]], label [[FPCLASSIFY_END]], label [[FPCLASSIFY_NOT_NAN]]
 ; CHECK:       fpclassify_not_nan:
 ; CHECK-NEXT:    [[X_ABS:%.*]] = tail call float @llvm.fabs.f32(float [[X]])
 ; CHECK-NEXT:    [[ISINF:%.*]] = fcmp oeq float [[X_ABS]], 0x7FF0000000000000
diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll b/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll
index 5a69a0c002fb8..65a5f5e8e138b 100644
--- a/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll
+++ b/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -S < %s | FileCheck %s
+; RUN: opt -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -simplifycfg-branch-fold-threshold=0 -S < %s | FileCheck %s
 
 define i32 @foo(i32 %i) nounwind ssp !dbg !0 {
 ; CHECK-LABEL: @foo(
diff --git a/llvm/test/Transforms/SimplifyCFG/speculate-derefable-load.ll b/llvm/test/Transforms/SimplifyCFG/speculate-derefable-load.ll
index b8c999d700aa7..1b7effa6ef1c1 100644
--- a/llvm/test/Transforms/SimplifyCFG/speculate-derefable-load.ll
+++ b/llvm/test/Transforms/SimplifyCFG/speculate-derefable-load.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S -passes=simplifycfg < %s | FileCheck %s
+; RUN: opt -S -passes=simplifycfg -simplifycfg-branch-fold-threshold=0 < %s | FileCheck %s
 
 define i64 @align_deref_align(i1 %c, ptr %p) {
 ; CHECK-LABEL: define i64 @align_deref_align(
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-on-const-select.ll b/llvm/test/Transforms/SimplifyCFG/switch-on-const-select.ll
index e8b58639c13dd..38bb84def8e00 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch-on-const-select.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch-on-const-select.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s
+; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -simplifycfg-branch-fold-threshold=0 -S | FileCheck %s
 
 ; Test basic folding to a conditional branch.
 define i32 @foo(i64 %x, i64 %y) nounwind {

>From 8e4cd1b26d7a24200dec00064725d805007016b8 Mon Sep 17 00:00:00 2001
From: h2h <h2h at meta.com>
Date: Mon, 16 Jun 2025 14:56:11 -0700
Subject: [PATCH 2/2] Formatting

---
 llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 36 ++++++++++++-----------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 3c4d14b169bfd..895bd37e093a2 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -3965,11 +3965,10 @@ shouldFoldCondBranchesToCommonDestination(BranchInst *BI, BranchInst *PBI,
   return std::nullopt;
 }
 
-static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
-                                             DomTreeUpdater *DTU,
-                                             MemorySSAUpdater *MSSAU,
-                                             const TargetTransformInfo *TTI,
-                                             SmallDenseMap<PHINode*, SelectInst*, 8> &InsertNewPHIs) {
+static bool performBranchToCommonDestFolding(
+    BranchInst *BI, BranchInst *PBI, DomTreeUpdater *DTU,
+    MemorySSAUpdater *MSSAU, const TargetTransformInfo *TTI,
+    SmallDenseMap<PHINode *, SelectInst *, 8> &InsertNewPHIs) {
   BasicBlock *BB = BI->getParent();
   BasicBlock *PredBlock = PBI->getParent();
 
@@ -4064,7 +4063,7 @@ static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
         Instruction *SI = It->second;
         // Oprands might have been promoted to bonous inst
         RemapInstruction(SI, VMap,
-                RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
+                         RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
         // Insert SelectInst as the new PHINode incoming value
         SI->insertBefore(PredBlock->getTerminator()->getIterator());
         // Fix PHINode
@@ -4139,11 +4138,11 @@ bool llvm::foldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
   SmallVector<BasicBlock *, 8> Preds;
   struct InsertPointTy {
     InstructionCost Cost;
-    Value          *TValue; // True Value
-    Value          *FValue; // False Value
-    PHINode        *Phi;
+    Value *TValue; // True Value
+    Value *FValue; // False Value
+    PHINode *Phi;
   };
-  SmallDenseMap<BranchInst*, SmallVector<InsertPointTy, 8>, 8> InsertPts;
+  SmallDenseMap<BranchInst *, SmallVector<InsertPointTy, 8>, 8> InsertPts;
   for (BasicBlock *PredBlock : predecessors(BB)) {
     BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator());
 
@@ -4151,8 +4150,9 @@ bool llvm::foldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
     if (!PBI || PBI->isUnconditional())
       continue;
 
-    // If there is a PHI node in the common successor, verify that the same value flows in from both
-    // blocks. Otherwise, check whether we can create a SelectInst to combine the incoming values
+    // If there is a PHI node in the common successor, verify that the same
+    // value flows in from both blocks. Otherwise, check whether we can create a
+    // SelectInst to combine the incoming values
     if (!safeToMergeTerminators(BI, PBI)) {
       if (BI == PBI)
         continue;
@@ -4163,9 +4163,10 @@ bool llvm::foldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
             Value *IV1 = Phi.getIncomingValueForBlock(PredBlock);
             InstructionCost PCost;
             if (TTI) {
-              PCost = TTI->getCmpSelInstrCost(Instruction::Select, Phi.getType(),
-                                     CmpInst::makeCmpResultType(Phi.getType()),
-                                     CmpInst::BAD_ICMP_PREDICATE, CostKind);
+              PCost = TTI->getCmpSelInstrCost(
+                  Instruction::Select, Phi.getType(),
+                  CmpInst::makeCmpResultType(Phi.getType()),
+                  CmpInst::BAD_ICMP_PREDICATE, CostKind);
             }
             auto &IP = InsertPts[PBI];
             if (PBI->getSuccessor(0) == BB)
@@ -4268,8 +4269,9 @@ bool llvm::foldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
     SmallDenseMap<PHINode *, SelectInst *, 8> newPhis;
     if (InsertPts.contains(PBI)) {
       Value *PC = PBI->getCondition();
-      for (auto const InsertInfo: InsertPts[PBI]) {
-        SelectInst *newPhi = SelectInst::Create(PC, InsertInfo.TValue, InsertInfo.FValue);
+      for (auto const InsertInfo : InsertPts[PBI]) {
+        SelectInst *newPhi =
+            SelectInst::Create(PC, InsertInfo.TValue, InsertInfo.FValue);
         newPhis.insert(std::make_pair(InsertInfo.Phi, newPhi));
       }
     }



More information about the llvm-commits mailing list