[llvm] r235942 - Switch lowering: Take branch weight into account when ordering for fall-through

[Hans Wennborg]

Author: hans
Date: Mon Apr 27 18:35:22 2015
New Revision: 235942

URL: http://llvm.org/viewvc/llvm-project?rev=235942&view=rev
Log:
Switch lowering: Take branch weight into account when ordering for fall-through

Previously, the code would try to put a fall-through case last,
even if that meant moving a case with much higher branch weight
further down the chain.

Ordering by branch weight is most important, putting a fall-through
block last is secondary.

Modified:
    llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
    llvm/trunk/test/CodeGen/ARM/arm-and-tst-peephole.ll
    llvm/trunk/test/CodeGen/Generic/MachineBranchProb.ll
    llvm/trunk/test/CodeGen/X86/ragreedy-hoist-spill.ll
    llvm/trunk/test/CodeGen/X86/switch.ll

Modified: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp?rev=235942&r1=235941&r2=235942&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (original)
+++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp Mon Apr 27 18:35:22 2015
@@ -7678,11 +7678,12 @@ void SelectionDAGBuilder::lowerWorkItem(
       return a.Weight > b.Weight;
     });
 
-    // Rearrange the case blocks so that the last one falls through if possible.
-    // Start at the bottom as that's the case with the lowest weight.
-    // FIXME: Take branch probability into account.
+    // Rearrange the case blocks so that the last one falls through if possible
+    // without without changing the order of weights.
     for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
       --I;
+      if (I->Weight > W.LastCluster->Weight)
+        break;
       if (I->Kind == CC_Range && I->MBB == NextMBB) {
         std::swap(*I, *W.LastCluster);
         break;

Modified: llvm/trunk/test/CodeGen/ARM/arm-and-tst-peephole.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/ARM/arm-and-tst-peephole.ll?rev=235942&r1=235941&r2=235942&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/ARM/arm-and-tst-peephole.ll (original)
+++ llvm/trunk/test/CodeGen/ARM/arm-and-tst-peephole.ll Mon Apr 27 18:35:22 2015
@@ -49,14 +49,14 @@ tailrecurse.switch:
 ; V8-NEXT: beq
 ; V8-NEXT: %tailrecurse.switch
 ; V8: cmp
-; V8-NEXT: beq
+; V8-NEXT: bne
 ; V8-NEXT: b	
 ; The trailing space in the last line checks that the branch is unconditional
   switch i32 %and, label %sw.epilog [
     i32 1, label %sw.bb
     i32 3, label %sw.bb6
     i32 2, label %sw.bb8
-  ]
+  ], !prof !1
 
 sw.bb:                                            ; preds = %tailrecurse.switch, %tailrecurse
   %shl = shl i32 %acc.tr, 1
@@ -134,3 +134,5 @@ bb4:
 return:                                           ; preds = %bb2, %bb, %entry
   ret i8 1
 }
+
+!1 = !{!"branch_weights", i32 1, i32 1, i32 3, i32 2 }

Modified: llvm/trunk/test/CodeGen/Generic/MachineBranchProb.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/Generic/MachineBranchProb.ll?rev=235942&r1=235941&r2=235942&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/Generic/MachineBranchProb.ll (original)
+++ llvm/trunk/test/CodeGen/Generic/MachineBranchProb.ll Mon Apr 27 18:35:22 2015
@@ -17,9 +17,9 @@ entry:
 ; CHECK: BB#0: derived from LLVM BB %entry
 ; CHECK: Successors according to CFG: BB#2(64) BB#4(14)
 ; CHECK: BB#4: derived from LLVM BB %entry
-; CHECK: Successors according to CFG: BB#1(4) BB#5(10)
+; CHECK: Successors according to CFG: BB#1(10) BB#5(4)
 ; CHECK: BB#5: derived from LLVM BB %entry
-; CHECK: Successors according to CFG: BB#1(10) BB#3(7)
+; CHECK: Successors according to CFG: BB#1(4) BB#3(7)
 
 sw.bb:
   br label %return

Modified: llvm/trunk/test/CodeGen/X86/ragreedy-hoist-spill.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/ragreedy-hoist-spill.ll?rev=235942&r1=235941&r2=235942&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/X86/ragreedy-hoist-spill.ll (original)
+++ llvm/trunk/test/CodeGen/X86/ragreedy-hoist-spill.ll Mon Apr 27 18:35:22 2015
@@ -30,7 +30,7 @@ if.then:
 if.end:
   switch i64 undef, label %if.end25 [
     i64 0, label %if.then4
-    i64 1, label %land.lhs.true14
+    i64 1, label %if.end25
   ]
 
 if.then4:

Modified: llvm/trunk/test/CodeGen/X86/switch.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/switch.ll?rev=235942&r1=235941&r2=235942&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/X86/switch.ll (original)
+++ llvm/trunk/test/CodeGen/X86/switch.ll Mon Apr 27 18:35:22 2015
@@ -413,3 +413,32 @@ return: ret void
        i32 4294967295, i32 2, i32 4294967295,
        ; Cases 2,5,8,9:
        i32 3, i32 3, i32 3, i32 3}
+
+define void @order_by_weight_and_fallthrough(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 100, label %bb1
+    i32 200, label %bb0
+    i32 300, label %bb0
+  ], !prof !2
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+return: ret void
+
+; Case 200 has the highest weight and should come first. 100 and 300 have the
+; same weight, but 300 goes to the 'next' block, so should be last.
+; CHECK-LABEL: order_by_weight_and_fallthrough
+; CHECK: cmpl $200
+; CHECK: cmpl $100
+; CHECK: cmpl $300
+}
+
+!2 = !{!"branch_weights",
+       ; Default:
+       i32 1,
+       ; Case 100:
+       i32 10,
+       ; Case 200:
+       i32 1000,
+       ; Case 300:
+       i32 10}