[llvm] [LoopPeel] Peel last iteration to enable load widening (PR #173420)
Guy David via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 10:46:00 PDT 2026
https://github.com/guy-david updated https://github.com/llvm/llvm-project/pull/173420
>From 16817382e75e984bbe7cd2b9042660154e172477 Mon Sep 17 00:00:00 2001
From: Guy David <guyda96 at gmail.com>
Date: Mon, 3 Aug 2026 10:57:32 +0200
Subject: [PATCH 1/2] [NFC][LoopPeel] Add tests for load widening
---
.../peel-last-iteration-load-widening-be.ll | 87 +
.../peel-last-iteration-load-widening.ll | 1717 +++++++++++++++++
2 files changed, 1804 insertions(+)
create mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-be.ll
create mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening.ll
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-be.ll b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-be.ll
new file mode 100644
index 0000000000000..fcb08a24a1895
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-be.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=aarch64_be -passes=loop-unroll -S %s | FileCheck %s
+; RUN: opt -mtriple=aarch64_be -passes=loop-unroll -S %s | FileCheck %s --check-prefix=IGNORE-COST
+
+define void @test_3_consecutive_loads_be(ptr %src, ptr %dst, i32 %n) {
+;
+;
+;
+;
+; CHECK-LABEL: define void @test_3_consecutive_loads_be(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META0:![0-9]+]]
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; IGNORE-COST-LABEL: define void @test_3_consecutive_loads_be(
+; IGNORE-COST-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; IGNORE-COST-NEXT: [[ENTRY:.*]]:
+; IGNORE-COST-NEXT: br label %[[LOOP:.*]]
+; IGNORE-COST: [[LOOP]]:
+; IGNORE-COST-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; IGNORE-COST-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; IGNORE-COST-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; IGNORE-COST-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; IGNORE-COST-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META0:![0-9]+]]
+; IGNORE-COST-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META0]]
+; IGNORE-COST-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META0]]
+; IGNORE-COST-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; IGNORE-COST-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; IGNORE-COST-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; IGNORE-COST-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; IGNORE-COST-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; IGNORE-COST-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; IGNORE-COST-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; IGNORE-COST-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; IGNORE-COST: [[EXIT]]:
+; IGNORE-COST-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1, !noundef !{}
+ %b = load i8, ptr %p1, align 1, !noundef !{}
+ %c = load i8, ptr %p2, align 1, !noundef !{}
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+;.
+; CHECK: [[META0]] = !{}
+;.
+; IGNORE-COST: [[META0]] = !{}
+;.
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening.ll b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening.ll
new file mode 100644
index 0000000000000..1854b77b4fe59
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening.ll
@@ -0,0 +1,1717 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=aarch64 -passes=loop-unroll -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p2:128:128:128-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+
+define void @test_3_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_3_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !noundef [[META0:![0-9]+]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1, !noundef !{}
+ %b = load i8, ptr %p1, align 1, !noundef !{}
+ %c = load i8, ptr %p2, align 1, !noundef !{}
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_5_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_5_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[TMP15:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP16:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 2
+; CHECK-NEXT: [[P3_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 3
+; CHECK-NEXT: [[P4_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 4
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP16]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[D_PEEL:%.*]] = load i8, ptr [[P3_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[E_PEEL:%.*]] = load i8, ptr [[P4_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[SUM3_PEEL:%.*]] = add i8 [[SUM2_PEEL]], [[D_PEEL]]
+; CHECK-NEXT: [[SUM4_PEEL:%.*]] = add i8 [[SUM3_PEEL]], [[E_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP15]]
+; CHECK-NEXT: store i8 [[SUM4_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP16]], i64 5
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP15]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %p3 = getelementptr inbounds i8, ptr %p, i64 3
+ %p4 = getelementptr inbounds i8, ptr %p, i64 4
+
+ %a = load i8, ptr %p, align 1, !noundef !{}
+ %b = load i8, ptr %p1, align 1, !noundef !{}
+ %c = load i8, ptr %p2, align 1, !noundef !{}
+ %d = load i8, ptr %p3, align 1, !noundef !{}
+ %e = load i8, ptr %p4, align 1, !noundef !{}
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %sum3 = add i8 %sum2, %d
+ %sum4 = add i8 %sum3, %e
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum4, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 5
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_8_consecutive_loads_no_peel(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_8_consecutive_loads_no_peel(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 5
+; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 6
+; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 7
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[D:%.*]] = load i8, ptr [[P3]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[E:%.*]] = load i8, ptr [[P4]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[F:%.*]] = load i8, ptr [[P5]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[G:%.*]] = load i8, ptr [[P6]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[H:%.*]] = load i8, ptr [[P7]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[SUM3:%.*]] = add i8 [[SUM2]], [[D]]
+; CHECK-NEXT: [[SUM4:%.*]] = add i8 [[SUM3]], [[E]]
+; CHECK-NEXT: [[SUM5:%.*]] = add i8 [[SUM4]], [[F]]
+; CHECK-NEXT: [[SUM6:%.*]] = add i8 [[SUM5]], [[G]]
+; CHECK-NEXT: [[SUM7:%.*]] = add i8 [[SUM6]], [[H]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM7]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 8
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %p3 = getelementptr inbounds i8, ptr %p, i64 3
+ %p4 = getelementptr inbounds i8, ptr %p, i64 4
+ %p5 = getelementptr inbounds i8, ptr %p, i64 5
+ %p6 = getelementptr inbounds i8, ptr %p, i64 6
+ %p7 = getelementptr inbounds i8, ptr %p, i64 7
+
+ %a = load i8, ptr %p, align 1, !noundef !{}
+ %b = load i8, ptr %p1, align 1, !noundef !{}
+ %c = load i8, ptr %p2, align 1, !noundef !{}
+ %d = load i8, ptr %p3, align 1, !noundef !{}
+ %e = load i8, ptr %p4, align 1, !noundef !{}
+ %f = load i8, ptr %p5, align 1, !noundef !{}
+ %g = load i8, ptr %p6, align 1, !noundef !{}
+ %h = load i8, ptr %p7, align 1, !noundef !{}
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %sum3 = add i8 %sum2, %d
+ %sum4 = add i8 %sum3, %e
+ %sum5 = add i8 %sum4, %f
+ %sum6 = add i8 %sum5, %g
+ %sum7 = add i8 %sum6, %h
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum7, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 8
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_intervening_store(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_intervening_store(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META0]]
+; CHECK-NEXT: store i8 42, ptr [[P1]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1, !noundef !{}
+ store i8 42, ptr %p1, align 1 ; Intervening store prevents optimization
+ %b = load i8, ptr %p1, align 1, !noundef !{}
+ %c = load i8, ptr %p2, align 1, !noundef !{}
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_3_consecutive_i16_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_3_consecutive_i16_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i16, ptr [[TMP12]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i16, ptr [[TMP12]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i16, ptr [[TMP12]], align 2, !noundef [[META0]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i16, ptr [[P1_PEEL]], align 2, !noundef [[META0]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i16, ptr [[P2_PEEL]], align 2, !noundef [[META0]]
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i16 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i16 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP11]]
+; CHECK-NEXT: store i16 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 2
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 6
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i16, ptr %p, i64 1
+ %p2 = getelementptr inbounds i16, ptr %p, i64 2
+
+ %a = load i16, ptr %p, align 2, !noundef !{}
+ %b = load i16, ptr %p1, align 2, !noundef !{}
+ %c = load i16, ptr %p2, align 2, !noundef !{}
+
+ %sum1 = add i16 %a, %b
+ %sum2 = add i16 %sum1, %c
+ %dst.i = getelementptr inbounds i16, ptr %dst, i32 %i
+ store i16 %sum2, ptr %dst.i, align 2
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 6
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_aa_metadata_preserved(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_aa_metadata_preserved(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !tbaa [[TBAA1:![0-9]+]], !noundef [[META0]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !tbaa [[TBAA1]], !noundef [[META0]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !tbaa [[TBAA1]], !noundef [[META0]]
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1, !tbaa !0, !noundef !{}
+ %b = load i8, ptr %p1, align 1, !tbaa !0, !noundef !{}
+ %c = load i8, ptr %p2, align 1, !tbaa !0, !noundef !{}
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Differing AA metadata is intersected.
+define void @test_aa_metadata_different(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_aa_metadata_different(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !tbaa [[TBAA1]], !noundef [[META0]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !tbaa [[TBAA4:![0-9]+]], !noundef [[META0]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !tbaa [[TBAA1]], !noundef [[META0]]
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1, !tbaa !0, !noundef !{}
+ %b = load i8, ptr %p1, align 1, !tbaa !3, !noundef !{}
+ %c = load i8, ptr %p2, align 1, !tbaa !0, !noundef !{}
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_negative_offsets(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_negative_offsets(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SRC_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 2
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi ptr [ [[SRC_OFFSET]], %[[ENTRY]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P_NEG2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 -2
+; CHECK-NEXT: [[P_NEG1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 -1
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[P_NEG2_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P_NEG1_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[TMP13]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP12]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP13]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP12]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %src.offset = getelementptr inbounds i8, ptr %src, i64 2
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src.offset, %entry ], [ %p.next, %loop ]
+
+ %p_neg2 = getelementptr inbounds i8, ptr %p, i64 -2
+ %p_neg1 = getelementptr inbounds i8, ptr %p, i64 -1
+
+ %a = load i8, ptr %p_neg2, align 1, !noundef !{}
+ %b = load i8, ptr %p_neg1, align 1, !noundef !{}
+ %c = load i8, ptr %p, align 1, !noundef !{}
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_non_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_non_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 5
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 2
+ %p2 = getelementptr inbounds i8, ptr %p, i64 4
+
+ %a = load i8, ptr %p, align 1, !noundef !{}
+ %b = load i8, ptr %p1, align 1, !noundef !{}
+ %c = load i8, ptr %p2, align 1, !noundef !{}
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 5
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_non_inbounds_geps(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_non_inbounds_geps(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr i8, ptr %p, i64 1
+ %p2 = getelementptr i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1, !noundef !{}
+ %b = load i8, ptr %p1, align 1, !noundef !{}
+ %c = load i8, ptr %p2, align 1, !noundef !{}
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Mixed element widths, including overlapping loads.
+define void @test_mixed_size_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_mixed_size_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i16, ptr [[P1_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[A_EXT_PEEL:%.*]] = zext i8 [[A_PEEL]] to i16
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i16 [[A_EXT_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[C_EXT_PEEL:%.*]] = zext i8 [[C_PEEL]] to i16
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i16 [[SUM1_PEEL]], [[C_EXT_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP11]]
+; CHECK-NEXT: store i16 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 2
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1, !noundef !{}
+ %b = load i16, ptr %p1, align 1, !noundef !{}
+ %c = load i8, ptr %p2, align 1, !noundef !{}
+
+ %a.ext = zext i8 %a to i16
+ %sum1 = add i16 %a.ext, %b
+ %c.ext = zext i8 %c to i16
+ %sum2 = add i16 %sum1, %c.ext
+ %dst.i = getelementptr inbounds i16, ptr %dst, i32 %i
+ store i16 %sum2, ptr %dst.i, align 2
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_atomic_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_atomic_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load atomic i8, ptr [[P]] monotonic, align 1
+; CHECK-NEXT: [[B:%.*]] = load atomic i8, ptr [[P1]] monotonic, align 1
+; CHECK-NEXT: [[C:%.*]] = load atomic i8, ptr [[P2]] monotonic, align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load atomic i8, ptr %p monotonic, align 1
+ %b = load atomic i8, ptr %p1 monotonic, align 1
+ %c = load atomic i8, ptr %p2 monotonic, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_step_mismatch(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_step_mismatch(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1, !noundef !{}
+ %b = load i8, ptr %p1, align 1, !noundef !{}
+ %c = load i8, ptr %p2, align 1, !noundef !{}
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 4
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_alignment_inferred_from_group(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_alignment_inferred_from_group(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 4, !noundef [[META0]]
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1, !noundef !{}
+ %b = load i8, ptr %p1, align 1, !noundef !{}
+ %c = load i8, ptr %p2, align 4, !noundef !{}
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; The loaded bytes index memory, so a poison value would be immediate UB.
+define void @test_loads_nonpoison_via_address_use(ptr %in, ptr %weights, ptr %out, i32 %n) {
+; CHECK-LABEL: define void @test_loads_nonpoison_via_address_use(
+; CHECK-SAME: ptr [[IN:%.*]], ptr [[WEIGHTS:%.*]], ptr [[OUT:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[IN]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[G_PTR_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
+; CHECK-NEXT: [[B_PTR_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
+; CHECK-NEXT: [[R_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1
+; CHECK-NEXT: [[G_PEEL:%.*]] = load i8, ptr [[G_PTR_PEEL]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[B_PTR_PEEL]], align 1
+; CHECK-NEXT: [[R32_PEEL:%.*]] = zext i8 [[R_PEEL]] to i32
+; CHECK-NEXT: [[G32_PEEL:%.*]] = zext i8 [[G_PEEL]] to i32
+; CHECK-NEXT: [[B32_PEEL:%.*]] = zext i8 [[B_PEEL]] to i32
+; CHECK-NEXT: [[S1_PEEL:%.*]] = add i32 [[R32_PEEL]], [[G32_PEEL]]
+; CHECK-NEXT: [[IDX_PEEL:%.*]] = add i32 [[S1_PEEL]], [[B32_PEEL]]
+; CHECK-NEXT: [[W_PTR_PEEL:%.*]] = getelementptr inbounds i8, ptr [[WEIGHTS]], i32 [[IDX_PEEL]]
+; CHECK-NEXT: [[W_PEEL:%.*]] = load i8, ptr [[W_PTR_PEEL]], align 1
+; CHECK-NEXT: [[OUT_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i32 [[TMP11]]
+; CHECK-NEXT: store i8 [[W_PEEL]], ptr [[OUT_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %in, %entry ], [ %p.next, %loop ]
+
+ %g.ptr = getelementptr inbounds i8, ptr %p, i64 1
+ %b.ptr = getelementptr inbounds i8, ptr %p, i64 2
+
+ %r = load i8, ptr %p, align 1
+ %g = load i8, ptr %g.ptr, align 1
+ %b = load i8, ptr %b.ptr, align 1
+
+ %r32 = zext i8 %r to i32
+ %g32 = zext i8 %g to i32
+ %b32 = zext i8 %b to i32
+ %s1 = add i32 %r32, %g32
+ %idx = add i32 %s1, %b32
+
+ %w.ptr = getelementptr inbounds i8, ptr %weights, i32 %idx
+ %w = load i8, ptr %w.ptr, align 1
+ %out.i = getelementptr inbounds i8, ptr %out, i32 %i
+ store i8 %w, ptr %out.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Span too wide for any legal integer type.
+define void @test_huge_offset(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_huge_offset(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PBIG:%.*]] = getelementptr inbounds i8, ptr [[P]], i128 100000000000000000000
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[PBIG]], align 1
+; CHECK-NEXT: [[SUM:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i128 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %pbig = getelementptr inbounds i8, ptr %p, i128 100000000000000000000
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %pbig, align 1
+
+ %sum = add i8 %a, %b
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i128 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; The call may not return, so the next iteration may not load these bytes.
+define void @test_call_may_not_return(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_call_may_not_return(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: call void @sink_maynotreturn(i8 [[SUM2]])
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ call void @sink_maynotreturn(i8 %sum2)
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_call_willreturn(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_call_willreturn(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: call void @sink_willreturn(i8 [[SUM2_PEEL]])
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ call void @sink_willreturn(i8 %sum2)
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_call_may_not_return_other_block(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_call_may_not_return_other_block(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: call void @sink_maynotreturn(i8 [[SUM2]])
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %latch ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %latch ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+ br label %latch
+
+latch:
+ call void @sink_maynotreturn(i8 %sum2)
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Offset needs more than 64 bits; it must not be sign-extended to int64_t.
+define void @test_offset_exceeds_int64(ptr addrspace(2) %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_offset_exceeds_int64(
+; CHECK-SAME: ptr addrspace(2) [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr addrspace(2) [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PBIG:%.*]] = getelementptr inbounds i8, ptr addrspace(2) [[P]], i128 100000000000000000000
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr addrspace(2) [[P]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr addrspace(2) [[PBIG]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[SUM:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr addrspace(2) [[P]], i128 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr addrspace(2) [ %src, %entry ], [ %p.next, %loop ]
+
+ %pbig = getelementptr inbounds i8, ptr addrspace(2) %p, i128 100000000000000000000
+
+ %a = load i8, ptr addrspace(2) %p, align 1, !noundef !{}
+ %b = load i8, ptr addrspace(2) %pbig, align 1, !noundef !{}
+
+ %sum = add i8 %a, %b
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr addrspace(2) %p, i128 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Peel-last chosen for the compare, so widenLoadsAfterPeel must re-check.
+define void @test_peel_last_for_compare_not_widened(ptr %src, i32 %n) {
+; CHECK-LABEL: define void @test_peel_last_for_compare_not_widened(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SUB:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ne i32 [[SUB]], 0
+; CHECK-NEXT: br i1 [[TMP0]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: call void @sink_maynotreturn(i8 [[SUM2]])
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i32 [[IV]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i32 [[IV_NEXT]], [[TMP1]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[IV_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP2:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[CMP_PEEL:%.*]] = icmp eq i32 [[TMP2]], [[SUB]]
+; CHECK-NEXT: [[SEL_PEEL:%.*]] = select i1 [[CMP_PEEL]], i8 1, i8 [[SUM2_PEEL]]
+; CHECK-NEXT: call void @sink_maynotreturn(i8 [[SEL_PEEL]])
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i64 3
+; CHECK-NEXT: [[IV_NEXT_PEEL:%.*]] = add i32 [[TMP2]], 1
+; CHECK-NEXT: [[EC_PEEL:%.*]] = icmp ne i32 [[IV_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[EC_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %sub = add i32 %n, -1
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+
+ %cmp = icmp eq i32 %iv, %sub
+ %sel = select i1 %cmp, i8 1, i8 %sum2
+ call void @sink_maynotreturn(i8 %sel)
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %iv.next = add i32 %iv, 1
+ %ec = icmp ne i32 %iv.next, %n
+ br i1 %ec, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Non-byte-multiple width that fills its store slot must not be rejected.
+define void @test_i24_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_i24_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 3
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i24, ptr [[TMP10]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i24, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[SUM_PEEL:%.*]] = add i24 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i24, ptr [[DST]], i32 [[TMP9]]
+; CHECK-NEXT: store i24 [[SUM_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP10]], i64 6
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+ %p1 = getelementptr inbounds i8, ptr %p, i64 3
+ %a = load i24, ptr %p, align 1
+ %b = load i24, ptr %p1, align 1
+ %sum = add i24 %a, %b
+ %dst.i = getelementptr inbounds i24, ptr %dst, i32 %i
+ store i24 %sum, ptr %dst.i, align 1
+ %p.next = getelementptr inbounds i8, ptr %p, i64 6
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; The base is a getelementptr with a variable index rather than a phi.
+define void @test_base_plus_iv_pattern(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_base_plus_iv_pattern(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[I_EXT_PEEL:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT: [[STRIDE_PEEL:%.*]] = mul i64 [[I_EXT_PEEL]], 3
+; CHECK-NEXT: [[P_PEEL:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[STRIDE_PEEL]]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[P_PEEL]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[P_PEEL]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[P_PEEL]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %i.ext = zext i32 %i to i64
+ %stride = mul i64 %i.ext, 3
+ %p = getelementptr inbounds i8, ptr %src, i64 %stride
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; A non-dominating group stays unwidened when another group forces the peel.
+define void @test_conditional_group_not_widened(ptr %pa, ptr %pb, ptr %dst, i32 %n, i1 %c0) {
+; CHECK-LABEL: define void @test_conditional_group_not_widened(
+; CHECK-SAME: ptr [[PA:%.*]], ptr [[PB:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]], i1 [[C0:%.*]]) {
+; CHECK-NEXT: [[EXIT_PEEL_BEGIN1:.*]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN1]] ], [ [[I_NEXT_PEEL:%.*]], %[[LATCH_PEEL:.*]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi ptr [ [[PA]], %[[EXIT_PEEL_BEGIN1]] ], [ [[A_NEXT_PEEL:%.*]], %[[LATCH_PEEL]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi ptr [ [[PB]], %[[EXIT_PEEL_BEGIN1]] ], [ [[B_NEXT_PEEL:%.*]], %[[LATCH_PEEL]] ]
+; CHECK-NEXT: [[A1_PEEL1:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 1
+; CHECK-NEXT: [[A2_PEEL1:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 2
+; CHECK-NEXT: [[AV0_PEEL1:%.*]] = load i8, ptr [[TMP14]], align 1
+; CHECK-NEXT: [[AV1_PEEL1:%.*]] = load i8, ptr [[A1_PEEL1]], align 1
+; CHECK-NEXT: [[AV2_PEEL1:%.*]] = load i8, ptr [[A2_PEEL1]], align 1
+; CHECK-NEXT: [[AS1_PEEL1:%.*]] = add i8 [[AV0_PEEL1]], [[AV1_PEEL1]]
+; CHECK-NEXT: [[AS2_PEEL1:%.*]] = add i8 [[AS1_PEEL1]], [[AV2_PEEL1]]
+; CHECK-NEXT: store i8 [[AS2_PEEL1]], ptr [[DST]], align 1
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN_PEEL:.*]], label %[[LATCH_PEEL]]
+; CHECK: [[THEN_PEEL]]:
+; CHECK-NEXT: [[B1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 1
+; CHECK-NEXT: [[B2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 2
+; CHECK-NEXT: [[BV0_PEEL:%.*]] = load i8, ptr [[TMP13]], align 1
+; CHECK-NEXT: [[BV1_PEEL:%.*]] = load i8, ptr [[B1_PEEL]], align 1
+; CHECK-NEXT: [[BV2_PEEL:%.*]] = load i8, ptr [[B2_PEEL]], align 1
+; CHECK-NEXT: [[BS1_PEEL:%.*]] = add i8 [[BV0_PEEL]], [[BV1_PEEL]]
+; CHECK-NEXT: [[BS2_PEEL:%.*]] = add i8 [[BS1_PEEL]], [[BV2_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
+; CHECK-NEXT: store i8 [[BS2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: br label %[[LATCH_PEEL]]
+; CHECK: [[LATCH_PEEL]]:
+; CHECK-NEXT: [[A_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP14]], i64 3
+; CHECK-NEXT: [[B_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP13]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL1]], label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %latch ]
+ %a = phi ptr [ %pa, %entry ], [ %a.next, %latch ]
+ %b = phi ptr [ %pb, %entry ], [ %b.next, %latch ]
+ %a1 = getelementptr inbounds i8, ptr %a, i64 1
+ %a2 = getelementptr inbounds i8, ptr %a, i64 2
+ %av0 = load i8, ptr %a, align 1
+ %av1 = load i8, ptr %a1, align 1
+ %av2 = load i8, ptr %a2, align 1
+ %as1 = add i8 %av0, %av1
+ %as2 = add i8 %as1, %av2
+ store i8 %as2, ptr %dst, align 1
+ br i1 %c0, label %then, label %latch
+
+then:
+ %b1 = getelementptr inbounds i8, ptr %b, i64 1
+ %b2 = getelementptr inbounds i8, ptr %b, i64 2
+ %bv0 = load i8, ptr %b, align 1
+ %bv1 = load i8, ptr %b1, align 1
+ %bv2 = load i8, ptr %b2, align 1
+ %bs1 = add i8 %bv0, %bv1
+ %bs2 = add i8 %bs1, %bv2
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %bs2, ptr %dst.i, align 1
+ br label %latch
+
+latch:
+ %a.next = getelementptr inbounds i8, ptr %a, i64 3
+ %b.next = getelementptr inbounds i8, ptr %b, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+declare void @sink_maynotreturn(i8)
+declare void @sink_willreturn(i8) willreturn nounwind
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"char", !2, i64 0}
+!2 = !{!"root"}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"int", !2, i64 0}
+
+; The wide load must be inserted at the first load in program order.
+define void @test_unordered_loads_use_between(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_unordered_loads_use_between(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[CZ_PEEL:%.*]] = zext i8 [[C_PEEL]] to i32
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[SZ_PEEL:%.*]] = zext i8 [[SUM2_PEEL]] to i32
+; CHECK-NEXT: [[TOT_PEEL:%.*]] = add i32 [[CZ_PEEL]], [[SZ_PEEL]]
+; CHECK-NEXT: [[T_PEEL:%.*]] = trunc i32 [[TOT_PEEL]] to i8
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
+; CHECK-NEXT: store i8 [[T_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %c = load i8, ptr %p2, align 1
+ %cz = zext i8 %c to i32
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %sz = zext i8 %sum2 to i32
+ %tot = add i32 %cz, %sz
+ %t = trunc i32 %tot to i8
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %t, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_base_addrspace_mismatch(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_base_addrspace_mismatch(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PC:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(1)
+; CHECK-NEXT: [[PC1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[PC]], i64 1
+; CHECK-NEXT: [[PC2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[PC]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr addrspace(1) [[PC]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr addrspace(1) [[PC1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr addrspace(1) [[PC2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %pc = addrspacecast ptr %p to ptr addrspace(1)
+ %pc1 = getelementptr inbounds i8, ptr addrspace(1) %pc, i64 1
+ %pc2 = getelementptr inbounds i8, ptr addrspace(1) %pc, i64 2
+ %a = load i8, ptr addrspace(1) %pc, align 1
+ %b = load i8, ptr addrspace(1) %pc1, align 1
+ %c = load i8, ptr addrspace(1) %pc2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_group_addrspace_mismatch(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_group_addrspace_mismatch(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PC:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(1)
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[PC2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[PC]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr addrspace(1) [[PC2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %pc = addrspacecast ptr %p to ptr addrspace(1)
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %pc2 = getelementptr inbounds i8, ptr addrspace(1) %pc, i64 2
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr addrspace(1) %pc2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_type_size_ne_store_size(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_type_size_ne_store_size(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i7, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i7, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i7, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i7 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i7 [[SUM1]], [[C]]
+; CHECK-NEXT: [[T:%.*]] = zext i7 [[SUM2]] to i8
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[T]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %a = load i7, ptr %p, align 1
+ %b = load i7, ptr %p1, align 1
+ %c = load i7, ptr %p2, align 1
+
+ %sum1 = add i7 %a, %b
+ %sum2 = add i7 %sum1, %c
+ %t = zext i7 %sum2 to i8
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %t, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_group_not_dominating_latch(ptr %src, ptr %dst, i32 %n, i1 %c0) {
+; CHECK-LABEL: define void @test_group_not_dominating_latch(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]], i1 [[C0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN:.*]], label %[[LATCH]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %latch ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %latch ]
+ br i1 %c0, label %then, label %latch
+
+then:
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+ br label %latch
+
+latch:
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_strict_align(ptr %src, ptr %dst, i32 %n) #0 {
+; CHECK-LABEL: define void @test_strict_align(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+attributes #0 = { "target-features"="+strict-align" }
+;.
+; CHECK: [[META0]] = !{}
+; CHECK: [[TBAA1]] = !{[[META2:![0-9]+]], [[META2]], i64 0}
+; CHECK: [[META2]] = !{!"char", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"root"}
+; CHECK: [[TBAA4]] = !{[[META5:![0-9]+]], [[META5]], i64 0}
+; CHECK: [[META5]] = !{!"int", [[META3]], i64 0}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META7:![0-9]+]]}
+; CHECK: [[META7]] = !{!"llvm.loop.peeled.count", i32 1}
+;.
>From f134fb4196652c57ce721b7f9a54bd67b43f9a83 Mon Sep 17 00:00:00 2001
From: Guy David <guyda96 at gmail.com>
Date: Mon, 3 Aug 2026 11:00:34 +0200
Subject: [PATCH 2/2] [LoopPeel] Peel last iteration to enable load widening
In loops that contain multiple consecutive small loads (e.g., 3 bytes
loading i8's), peeling the last iteration makes it safe to read beyond
the accessed region, enabling the use of a wider load (e.g., b32) for
all other N-1 iterations.
Patterns such as:
```
%a = load i8, ptr %p
%b = load i8, ptr %p+1
%c = load i8, ptr %p+2
...
%p.next = getelementptr i8, ptr %p, 3
```
Can be transformed to:
```
%wide = load b32, ptr %p ; Read 4 bytes
%frozen = freeze b32 %wide
%value = bitcast b32 %frozen to i32
%a = trunc
%b = lshr + trunc
%c = lshr + trunc
%p.next = getelementptr i8, ptr %p, 3
...
```
The wide load is of a byte type rather than an integer to handle poison
values correctly. Byte values carry poison per bit instead of per value,
so the read-ahead bytes, which the peel makes dereferenceable, cannot
taint the grouped bytes.
This acts as a fallback strategy when vectorization fails and has
significant performance uplifts, most notably in image processing
where processing an RGB buffer is common.
Assisted-by: Opus 4.8
---
.../llvm/Analysis/TargetTransformInfo.h | 4 +
llvm/include/llvm/Transforms/Utils/LoopPeel.h | 6 +
llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 6 +
llvm/lib/Transforms/Utils/LoopPeel.cpp | 334 +++++++-
.../peel-last-iteration-load-widening-be.ll | 67 +-
...l-last-iteration-load-widening-disabled.ll | 55 ++
.../peel-last-iteration-load-widening.ll | 783 +++++++++++++++---
7 files changed, 1098 insertions(+), 157 deletions(-)
create mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-disabled.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 107ae4dba5075..2163c904c9399 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -792,6 +792,10 @@ class TargetTransformInfo {
/// Peel off the last PeelCount loop iterations.
bool PeelLast;
+
+ /// Allow peeling the last iteration to enable widening of consecutive
+ /// loads.
+ bool AllowLoadWideningPeel;
};
/// Get target-customized preferences for the generic loop peeling
diff --git a/llvm/include/llvm/Transforms/Utils/LoopPeel.h b/llvm/include/llvm/Transforms/Utils/LoopPeel.h
index 4b6f0c63251e2..6b9adc4893185 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopPeel.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopPeel.h
@@ -50,6 +50,12 @@ LLVM_ABI void computePeelCount(Loop *L, unsigned LoopSize,
AssumptionCache *AC = nullptr,
unsigned Threshold = UINT_MAX);
+/// Combine load instructions in a loop into a wider one, given that we peeled
+/// the last iteration and can assume the bytes are dereferenceable.
+LLVM_ABI bool widenLoadsAfterPeel(Loop &L, ScalarEvolution &SE,
+ const TargetTransformInfo &TTI,
+ DominatorTree &DT);
+
} // end namespace llvm
#endif // LLVM_TRANSFORMS_UTILS_LOOPPEEL_H
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index ad3b123f3327c..3e2560e4bd2fa 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -1380,6 +1380,10 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
MaxOrZero = SE.isBackedgeTakenCountMaxOrZero(L);
}
+ // Only allow peeling the last iteration for load widening after
+ // vectorization, so that loops which could be vectorized instead are not
+ // peeled.
+ PP.AllowLoadWideningPeel = !OnlyFullUnroll;
// computeUnrollCount() decides whether it is beneficial to use upper bound to
// fully unroll the loop.
unsigned Count =
@@ -1414,6 +1418,8 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
ValueToValueMapTy VMap;
peelLoop(L, PP.PeelCount, PP.PeelLast, LI, &SE, DT, &AC, PreserveLCSSA,
VMap);
+ if (PP.PeelLast && PP.AllowLoadWideningPeel)
+ widenLoadsAfterPeel(*L, SE, TTI, DT);
simplifyLoopAfterUnroll(L, true, LI, &SE, &DT, &AC, &TTI, L->getBlocks(),
nullptr);
// If the loop was peeled, we already "used up" the profile information
diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp
index cc45a3e09bd88..0f19dbd12d854 100644
--- a/llvm/lib/Transforms/Utils/LoopPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -21,6 +21,7 @@
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
@@ -45,6 +46,7 @@
#include <algorithm>
#include <cassert>
#include <cstdint>
+#include <limits>
#include <optional>
using namespace llvm;
@@ -88,6 +90,17 @@ static cl::opt<bool> EnablePeelingForIV(
"enable-peeling-for-iv", cl::init(false), cl::Hidden,
cl::desc("Enable peeling to convert Phi nodes into IVs"));
+static cl::opt<bool> EnablePeelForLoadWidening(
+ "enable-peel-for-load-widening", cl::init(true), cl::Hidden,
+ cl::desc(
+ "Enable peeling last iteration to enable consecutive load widening"));
+
+static cl::opt<bool> PeelForLoadWideningIgnoreCost(
+ "peel-load-widening-ignore-cost", cl::init(false), cl::Hidden,
+ cl::desc("Widen consecutive loads after peeling regardless of whether the "
+ "wide load and its extractions are cheaper than the loads they "
+ "replace"));
+
static const char *PeeledCountMetaData = "llvm.loop.peeled.count";
extern cl::opt<bool> ProfcheckDisableMetadataFixes;
@@ -507,16 +520,12 @@ bool llvm::canPeelLastIteration(const Loop &L, ScalarEvolution &SE) {
m_scev_AffineAddRec(m_SCEV(), m_scev_One(), m_SpecificLoop(&L)));
}
-/// Returns true if the last iteration can be peeled off and the condition (Pred
-/// LeftAR, RightSCEV) is known at the last iteration and the inverse condition
-/// is known at the second-to-last.
+/// Returns true if the condition (Pred LeftAR, RightSCEV) is known at the last
+/// iteration and the inverse condition is known at the second-to-last.
static bool shouldPeelLastIteration(Loop &L, CmpPredicate Pred,
const SCEVAddRecExpr *LeftAR,
const SCEV *RightSCEV, ScalarEvolution &SE,
const TargetTransformInfo &TTI) {
- if (!canPeelLastIteration(L, SE))
- return false;
-
const SCEV *BTC = SE.getBackedgeTakenCount(&L);
SCEVExpander Expander(SE, "loop-peel");
if (!SE.isKnownNonZero(BTC) &&
@@ -549,7 +558,7 @@ static bool shouldPeelLastIteration(Loop &L, CmpPredicate Pred,
// }
static std::pair<unsigned, unsigned>
countToEliminateCompares(Loop &L, unsigned MaxPeelCount, ScalarEvolution &SE,
- const TargetTransformInfo &TTI) {
+ const TargetTransformInfo &TTI, bool CanPeelLast) {
assert(L.isLoopSimplifyForm() && "Loop needs to be in loop simplify form");
unsigned DesiredPeelCount = 0;
unsigned DesiredPeelCountLast = 0;
@@ -637,7 +646,8 @@ countToEliminateCompares(Loop &L, unsigned MaxPeelCount, ScalarEvolution &SE,
const SCEV *Step = LeftAR->getStepRecurrence(SE);
if (!PeelWhilePredicateIsKnown(NewPeelCount, IterVal, RightSCEV, Step,
Pred)) {
- if (shouldPeelLastIteration(L, Pred, LeftAR, RightSCEV, SE, TTI))
+ if (CanPeelLast &&
+ shouldPeelLastIteration(L, Pred, LeftAR, RightSCEV, SE, TTI))
DesiredPeelCountLast = 1;
return;
}
@@ -696,7 +706,8 @@ countToEliminateCompares(Loop &L, unsigned MaxPeelCount, ScalarEvolution &SE,
SE.getConstant(AddRec->getType(), NewPeelCount), SE);
if (!PeelWhilePredicateIsKnown(NewPeelCount, IterVal, BoundSCEV, Step,
Pred)) {
- if (shouldPeelLastIteration(L, Pred, AddRec, BoundSCEV, SE, TTI))
+ if (CanPeelLast &&
+ shouldPeelLastIteration(L, Pred, AddRec, BoundSCEV, SE, TTI))
DesiredPeelCountLast = 1;
return;
}
@@ -749,6 +760,223 @@ static bool violatesLegacyMultiExitLoopCheck(Loop *L) {
});
}
+namespace {
+// Represents a group of loads in a loop that can be combined into a wider one.
+struct LoadGroup {
+ // Base object being read.
+ Value *BasePtr;
+ // First load instruction in the program order.
+ LoadInst *FirstLoad;
+ // Pairs of (load instruction, offset from base) sorted by offset.
+ SmallVector<std::pair<LoadInst *, int64_t>, 4> Loads;
+ // The byte type to load the whole group as.
+ Type *WideType;
+};
+
+// Compute the bit offset to extract an element from a packed integer,
+// accounting for endianness.
+static uint64_t getElementBitOffset(const DataLayout &DL, uint64_t BitPosition,
+ uint64_t ElementBits,
+ uint64_t ContainerBits) {
+ return DL.isBigEndian() ? ContainerBits - ElementBits - BitPosition
+ : BitPosition;
+}
+
+// Helper to compute load group span and validate for widening.
+static std::optional<LoadGroup> tryFormLoadGroupForWidening(
+ Value *Base, SmallVectorImpl<std::pair<LoadInst *, int64_t>> &Loads,
+ Loop &L, ScalarEvolution &SE, const DataLayout &DL,
+ const TargetTransformInfo &TTI) {
+ // All loads must share one address space.
+ unsigned AddrSpace = Loads[0].first->getPointerAddressSpace();
+ if (Base->getType()->getPointerAddressSpace() != AddrSpace)
+ return std::nullopt;
+ if (any_of(make_first_range(Loads), [AddrSpace](LoadInst *Load) {
+ return Load->getPointerAddressSpace() != AddrSpace;
+ }))
+ return std::nullopt;
+ // Extraction assumes each loaded value fills its store slot exactly.
+ if (any_of(make_first_range(Loads), [&DL](LoadInst *Load) {
+ return !DL.typeSizeEqualsStoreSize(Load->getType());
+ }))
+ return std::nullopt;
+ // Find the span of the loaded data.
+ int64_t Left = std::numeric_limits<int64_t>::max();
+ int64_t Right = std::numeric_limits<int64_t>::min();
+ for (const auto &[Load, OffsetVal] : Loads) {
+ int64_t StoreSize =
+ static_cast<int64_t>(DL.getTypeStoreSize(Load->getType()));
+ if (OffsetVal > std::numeric_limits<int64_t>::max() - StoreSize)
+ return std::nullopt;
+ Left = std::min(Left, OffsetVal);
+ Right = std::max(Right, OffsetVal + StoreSize);
+ }
+ assert((Left < Right) && "Invalid load group span");
+ uint64_t TotalBytes = Right - Left;
+ uint64_t TotalBits = TotalBytes * 8;
+ LLVMContext &Ctx = L.getHeader()->getContext();
+ // Peeling the last iteration makes this many bytes dereferenceable even for
+ // the group at the tail of the buffer, so a single natural-width access is
+ // safe.
+ unsigned NaturalBits = PowerOf2Ceil(TotalBits);
+ assert(NaturalBits <= TotalBits * 2 && "Read-ahead exceeds one group span");
+ // Skip an already natural sized group (does not need read-ahead).
+ if (NaturalBits == TotalBits)
+ return std::nullopt;
+ // A legal natural integer type must exist for the widened access.
+ Type *NaturalType = DL.getSmallestLegalIntType(Ctx, TotalBits);
+ if (!NaturalType || NaturalType->getIntegerBitWidth() != NaturalBits)
+ return std::nullopt;
+ // Load the whole natural width span as a byte type. Byte values carry
+ // poison per bit rather than per value, which is useful because the
+ // read-ahead could have tainted the grouped bytes.
+ Type *WideType = ByteType::get(Ctx, NaturalBits);
+ unsigned WideBits = NaturalBits;
+ // Check the width access is unconstrained and without penalty.
+ unsigned Fast = 0;
+ if (!TTI.allowsMisalignedMemoryAccesses(Ctx, NaturalBits, AddrSpace, Align(1),
+ &Fast) ||
+ !Fast)
+ return std::nullopt;
+ // Validate pointer stride across iterations is constant to prove read-ahead
+ // dereferenceability.
+ if (!match(SE.getSCEV(Base),
+ m_scev_AffineAddRec(m_SCEV(), m_scev_SpecificSInt(TotalBytes),
+ m_SpecificLoop(&L))))
+ return std::nullopt;
+
+ // The insertion point is the first load in program order, which is not
+ // necessarily the lowest offset one, so capture it before sorting.
+ LoadInst *FirstLoad = Loads[0].first;
+ // Sort by offset.
+ llvm::sort(Loads, llvm::less_second());
+ int64_t LowestOffset = Loads[0].second;
+
+ // The loads must fully tile the span [Left, Right) with no gaps. Gaps are not
+ // proven to be dereferenceable.
+ int64_t Covered = Left;
+ for (const auto &[Load, OffsetVal] : Loads) {
+ if (OffsetVal > Covered)
+ return std::nullopt;
+ Covered = std::max(
+ Covered,
+ OffsetVal + static_cast<int64_t>(DL.getTypeStoreSize(Load->getType())));
+ }
+ assert(Covered == Right && "Load group should tile its span");
+
+ TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ // Cost of original individual loads.
+ InstructionCost OriginalCost = 0;
+ for (const auto &[Load, OffsetVal] : Loads) {
+ OriginalCost += TTI.getMemoryOpCost(Instruction::Load, Load->getType(),
+ Load->getAlign(), AddrSpace, CostKind);
+ }
+ // Cost of the wide load + extraction operations.
+ InstructionCost WideLoadCost = TTI.getMemoryOpCost(
+ Instruction::Load, NaturalType, Align(1), AddrSpace, CostKind);
+ InstructionCost WidenedCost = WideLoadCost;
+ for (const auto &[Load, OffsetVal] : Loads) {
+ unsigned LoadBits = Load->getType()->getScalarSizeInBits();
+ uint64_t BitPosition = (OffsetVal - LowestOffset) * 8;
+ uint64_t BitOffset =
+ getElementBitOffset(DL, BitPosition, LoadBits, WideBits);
+ if (BitOffset != 0)
+ WidenedCost +=
+ TTI.getArithmeticInstrCost(Instruction::LShr, NaturalType, CostKind);
+ if (LoadBits < WideBits)
+ WidenedCost +=
+ TTI.getCastInstrCost(Instruction::Trunc, Load->getType(), NaturalType,
+ TTI::CastContextHint::None, CostKind);
+ }
+
+ LLVM_DEBUG(dbgs() << "Load widening cost: original=" << OriginalCost
+ << " wide load=" << WideLoadCost
+ << " widened total=" << WidenedCost << "\n");
+ if (!PeelForLoadWideningIgnoreCost && WidenedCost > OriginalCost)
+ return std::nullopt;
+
+ return LoadGroup{Base, FirstLoad, std::move(Loads), WideType};
+}
+
+// Find groups of consecutive loads in a basic block for peeling purposes
+static SmallVector<LoadGroup>
+findLoadGroupsForWidening(BasicBlock *BB, Loop &L, ScalarEvolution &SE,
+ const DataLayout &DL,
+ const TargetTransformInfo &TTI) {
+ SmallVector<LoadGroup> Groups;
+ // Mapping from base pointer to loads instructions and their offset from the
+ // base.
+ DenseMap<Value *, SmallVector<std::pair<LoadInst *, int64_t>>> LoadsByBase;
+
+ auto ProcessCollectedLoads = [&]() {
+ for (auto &[Base, Loads] : LoadsByBase) {
+ if (Loads.size() <= 1)
+ continue;
+ if (auto Group = tryFormLoadGroupForWidening(Base, Loads, L, SE, DL, TTI))
+ Groups.emplace_back(*std::move(Group));
+ }
+ LoadsByBase.clear();
+ };
+
+ for (Instruction &I : *BB) {
+ if (auto *Load = dyn_cast<LoadInst>(&I)) {
+ if (Load->isVolatile() || Load->isAtomic()) {
+ ProcessCollectedLoads();
+ continue;
+ }
+ if (!Load->getType()->isIntegerTy())
+ continue;
+ Value *Ptr = Load->getPointerOperand();
+ unsigned AddrSpace = Load->getPointerAddressSpace();
+ APInt Offset(DL.getIndexSizeInBits(AddrSpace), 0);
+ Value *ActualBase = Ptr->stripAndAccumulateConstantOffsets(
+ DL, Offset, /*AllowNonInbounds=*/false);
+ std::optional<int64_t> OffsetVal = Offset.trySExtValue();
+ if (!OffsetVal)
+ continue;
+ LoadsByBase[ActualBase].emplace_back(Load, *OffsetVal);
+ } else if (I.mayHaveSideEffects()) {
+ // Side effects may clobber memory, so we can only group loads that
+ // appear between side effects. Process what we've collected so far.
+ ProcessCollectedLoads();
+ }
+ }
+ ProcessCollectedLoads();
+ return Groups;
+}
+
+// Collect the load groups in a loop that peeling the last iteration would make
+// widenable. Returns an empty list if the loop itself is not eligible.
+static SmallVector<LoadGroup>
+collectWidenableLoadGroups(Loop &L, ScalarEvolution &SE,
+ const TargetTransformInfo &TTI, DominatorTree &DT) {
+ SmallVector<LoadGroup> Groups;
+ BasicBlock *Latch = L.getLoopLatch();
+ if (!EnablePeelForLoadWidening || !L.isInnermost() || !Latch)
+ return Groups;
+ if (Latch != L.getExitingBlock() || !all_of(L.blocks(), [](BasicBlock *BB) {
+ return isGuaranteedToTransferExecutionToSuccessor(BB);
+ }))
+ return Groups;
+ const DataLayout &DL = L.getHeader()->getDataLayout();
+ for (BasicBlock *BB : L.blocks()) {
+ // Only groups in blocks that dominate the latch execute on every iteration,
+ // so only those are guaranteed to be re-read by the next one.
+ if (!DT.dominates(BB, Latch))
+ continue;
+ append_range(Groups, findLoadGroupsForWidening(BB, L, SE, DL, TTI));
+ }
+ return Groups;
+}
+
+// Returns true if peeling the last iteration would enable widening load groups
+// to natural sizes. Returns false otherwise.
+static bool peelLastForLoadWidening(Loop &L, ScalarEvolution &SE,
+ const TargetTransformInfo &TTI,
+ DominatorTree &DT) {
+ return !collectWidenableLoadGroups(L, SE, TTI, DT).empty();
+}
+} // anonymous namespace
// Return the number of iterations we want to peel off.
void llvm::computePeelCount(Loop *L, unsigned LoopSize,
@@ -804,6 +1032,9 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
// in TTI.getPeelingPreferences or by the flag -unroll-peel-count.
unsigned DesiredPeelCount = TargetPeelCount;
+ // Check once if we can peel the last iteration to avoid repeated checks.
+ bool CanPeelLast = canPeelLastIteration(*L, SE);
+
// Here we try to get rid of Phis which become invariants or inductions after
// 1, 2, ..., N iterations of the loop. For this we compute the number for
// iterations after which every Phi is guaranteed to become an invariant or an
@@ -818,7 +1049,7 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
}
const auto &[CountToEliminateCmps, CountToEliminateCmpsLast] =
- countToEliminateCompares(*L, MaxPeelCount, SE, TTI);
+ countToEliminateCompares(*L, MaxPeelCount, SE, TTI, CanPeelLast);
DesiredPeelCount = std::max(DesiredPeelCount, CountToEliminateCmps);
if (DesiredPeelCount == 0)
@@ -855,6 +1086,21 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
}
}
+ // Check for consecutive load widening opportunity.
+ // Skip this when running before vectorization (AllowLoadWideningPeel=false)
+ // to avoid peeling loops that could have been vectorized instead.
+ if (PP.PeelCount == 0 && PP.AllowLoadWideningPeel && CanPeelLast &&
+ peelLastForLoadWidening(*L, SE, TTI, DT)) {
+ if (1 + AlreadyPeeled <= UnrollPeelMaxCount) {
+ LLVM_DEBUG(dbgs() << "Peel last iteration to enable consecutive load "
+ "widening.\n");
+ PP.PeelCount = 1;
+ PP.PeelProfiledIterations = false;
+ PP.PeelLast = true;
+ return;
+ }
+ }
+
// Bail if we know the statically calculated trip count.
// In this case we rather prefer partial unrolling.
if (TripCount)
@@ -895,6 +1141,73 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
}
}
+bool llvm::widenLoadsAfterPeel(Loop &L, ScalarEvolution &SE,
+ const TargetTransformInfo &TTI,
+ DominatorTree &DT) {
+ assert(L.getLoopLatch() && "Loop should have a latch after peeling");
+ const DataLayout &DL = L.getHeader()->getDataLayout();
+ bool Changed = false;
+
+ for (const LoadGroup &Group : collectWidenableLoadGroups(L, SE, TTI, DT)) {
+ IRBuilder<> Builder(Group.FirstLoad);
+ Value *BasePtr = Group.BasePtr;
+ int64_t FirstOffset = Group.Loads[0].second;
+ unsigned AddrSpace = Group.FirstLoad->getPointerAddressSpace();
+ // If the first load doesn't start at offset 0, we need to adjust.
+ if (FirstOffset != 0) {
+ Value *OrigPtr = Group.BasePtr;
+ BasePtr = Builder.CreatePtrAdd(
+ OrigPtr,
+ ConstantInt::get(Builder.getIndexTy(DL, AddrSpace), FirstOffset));
+ }
+ // Merge AA metadata from all loads using intersection for correctness.
+ AAMDNodes AATags = Group.FirstLoad->getAAMetadata();
+ for (LoadInst *Load : make_first_range(Group.Loads)) {
+ if (Load != Group.FirstLoad)
+ AATags = AATags.merge(Load->getAAMetadata());
+ }
+ // Infer the best alignment for the wider load from the individual loads.
+ Align WideAlign = Group.Loads[0].first->getAlign();
+ for (const auto &[Load, OffsetVal] : Group.Loads) {
+ uint64_t Delta = OffsetVal - FirstOffset;
+ WideAlign = std::max(WideAlign, commonAlignment(Load->getAlign(), Delta));
+ }
+ // Load the whole span as a byte type. The read-ahead bytes are
+ // dereferenceable because the last iteration was peeled.
+ LoadInst *WideLoad =
+ Builder.CreateAlignedLoad(Group.WideType, BasePtr, WideAlign);
+ unsigned SizeInBits = WideLoad->getType()->getScalarSizeInBits();
+ if (AATags)
+ WideLoad->setAAMetadata(AATags);
+ Type *IntTy = IntegerType::get(WideLoad->getContext(), SizeInBits);
+ Value *WideInt =
+ Builder.CreateBitCast(Builder.CreateFreeze(WideLoad), IntTy);
+ // For each original load, extract the corresponding bytes.
+ for (const auto &[Load, OffsetVal] : Group.Loads) {
+ unsigned LoadBits = Load->getType()->getScalarSizeInBits();
+ Value *Extracted = WideInt;
+ uint64_t BitPosition = (OffsetVal - FirstOffset) * 8;
+ uint64_t BitOffset =
+ getElementBitOffset(DL, BitPosition, LoadBits, SizeInBits);
+ if (BitOffset != 0)
+ Extracted =
+ Builder.CreateLShr(Extracted, ConstantInt::get(IntTy, BitOffset));
+ if (LoadBits < SizeInBits)
+ Extracted = Builder.CreateTrunc(Extracted, Load->getType());
+ Load->replaceAllUsesWith(Extracted);
+ }
+ // Delete the original loads.
+ for (LoadInst *Load : make_first_range(Group.Loads))
+ Load->eraseFromParent();
+
+ LLVM_DEBUG(dbgs() << "Widened " << Group.Loads.size()
+ << " loads into a single " << *WideLoad << "\n");
+ Changed = true;
+ }
+
+ return Changed;
+}
+
/// Clones the body of the loop L, putting it between \p InsertTop and \p
/// InsertBot.
/// \param IterNumber The serial number of the iteration currently being
@@ -1072,6 +1385,7 @@ llvm::gatherPeelingPreferences(Loop *L, ScalarEvolution &SE,
PP.AllowLoopNestsPeeling = false;
PP.PeelLast = false;
PP.PeelProfiledIterations = true;
+ PP.AllowLoadWideningPeel = true;
// Get the target specifc values.
TTI.getPeelingPreferences(L, SE, PP);
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-be.ll b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-be.ll
index fcb08a24a1895..a4a1637a4a822 100644
--- a/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-be.ll
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-be.ll
@@ -1,12 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -mtriple=aarch64_be -passes=loop-unroll -S %s | FileCheck %s
-; RUN: opt -mtriple=aarch64_be -passes=loop-unroll -S %s | FileCheck %s --check-prefix=IGNORE-COST
+; RUN: opt -mtriple=aarch64_be -passes=loop-unroll -peel-load-widening-ignore-cost -S %s | FileCheck %s --check-prefix=IGNORE-COST
define void @test_3_consecutive_loads_be(ptr %src, ptr %dst, i32 %n) {
;
;
-;
-;
; CHECK-LABEL: define void @test_3_consecutive_loads_be(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
@@ -33,23 +31,58 @@ define void @test_3_consecutive_loads_be(ptr %src, ptr %dst, i32 %n) {
; IGNORE-COST-LABEL: define void @test_3_consecutive_loads_be(
; IGNORE-COST-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; IGNORE-COST-NEXT: [[ENTRY:.*]]:
+; IGNORE-COST-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; IGNORE-COST-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; IGNORE-COST-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; IGNORE-COST: [[ENTRY_SPLIT]]:
; IGNORE-COST-NEXT: br label %[[LOOP:.*]]
; IGNORE-COST: [[LOOP]]:
-; IGNORE-COST-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
-; IGNORE-COST-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
-; IGNORE-COST-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
-; IGNORE-COST-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
-; IGNORE-COST-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META0:![0-9]+]]
-; IGNORE-COST-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META0]]
-; IGNORE-COST-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META0]]
-; IGNORE-COST-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
-; IGNORE-COST-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; IGNORE-COST-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; IGNORE-COST-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; IGNORE-COST-NEXT: [[TMP2:%.*]] = load b32, ptr [[P]], align 1
+; IGNORE-COST-NEXT: [[TMP3:%.*]] = freeze b32 [[TMP2]]
+; IGNORE-COST-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP3]] to i32
+; IGNORE-COST-NEXT: [[TMP5:%.*]] = lshr i32 [[TMP4]], 24
+; IGNORE-COST-NEXT: [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
+; IGNORE-COST-NEXT: [[TMP7:%.*]] = lshr i32 [[TMP4]], 16
+; IGNORE-COST-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP7]] to i8
+; IGNORE-COST-NEXT: [[TMP9:%.*]] = lshr i32 [[TMP4]], 8
+; IGNORE-COST-NEXT: [[TMP10:%.*]] = trunc i32 [[TMP9]] to i8
+; IGNORE-COST-NEXT: [[SUM1:%.*]] = add i8 [[TMP6]], [[TMP8]]
+; IGNORE-COST-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP10]]
; IGNORE-COST-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
; IGNORE-COST-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
; IGNORE-COST-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
-; IGNORE-COST-NEXT: [[I_NEXT]] = add i32 [[I]], 1
-; IGNORE-COST-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
-; IGNORE-COST-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; IGNORE-COST-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; IGNORE-COST-NEXT: [[TMP11:%.*]] = sub i32 [[N]], 1
+; IGNORE-COST-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP11]]
+; IGNORE-COST-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; IGNORE-COST: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; IGNORE-COST-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; IGNORE-COST-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; IGNORE-COST-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; IGNORE-COST: [[EXIT_PEEL_BEGIN]]:
+; IGNORE-COST-NEXT: [[TMP12:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; IGNORE-COST-NEXT: [[TMP13:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; IGNORE-COST-NEXT: br label %[[LOOP_PEEL:.*]]
+; IGNORE-COST: [[LOOP_PEEL]]:
+; IGNORE-COST-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 1
+; IGNORE-COST-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 2
+; IGNORE-COST-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP13]], align 1, !noundef [[META2:![0-9]+]]
+; IGNORE-COST-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !noundef [[META2]]
+; IGNORE-COST-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !noundef [[META2]]
+; IGNORE-COST-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; IGNORE-COST-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; IGNORE-COST-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP12]]
+; IGNORE-COST-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; IGNORE-COST-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 3
+; IGNORE-COST-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP12]], 1
+; IGNORE-COST-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; IGNORE-COST-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; IGNORE-COST: [[EXIT_PEEL_NEXT]]:
+; IGNORE-COST-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; IGNORE-COST: [[LOOP_PEEL_NEXT]]:
+; IGNORE-COST-NEXT: br label %[[EXIT:.*]]
; IGNORE-COST: [[EXIT]]:
; IGNORE-COST-NEXT: ret void
;
@@ -83,5 +116,7 @@ exit:
;.
; CHECK: [[META0]] = !{}
;.
-; IGNORE-COST: [[META0]] = !{}
+; IGNORE-COST: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; IGNORE-COST: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
+; IGNORE-COST: [[META2]] = !{}
;.
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-disabled.ll b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-disabled.ll
new file mode 100644
index 0000000000000..8d93a15bcd6cb
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-disabled.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=aarch64 -passes=loop-unroll -enable-peel-for-load-widening=false -S %s | FileCheck %s
+; RUN: opt -mtriple=aarch64 -passes=loop-unroll-full -S %s | FileCheck %s
+
+define void @test_3_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_3_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening.ll b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening.ll
index 1854b77b4fe59..e7f60f217d473 100644
--- a/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening.ll
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening.ll
@@ -7,24 +7,58 @@ define void @test_3_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
; CHECK-LABEL: define void @test_3_consecutive_loads(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN1:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load b32, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = freeze b32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP4]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[TMP4]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP8]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP9]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP10]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN1]]
+; CHECK: [[EXIT_PEEL_BEGIN1]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !noundef [[META0:![0-9]+]]
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !noundef [[META2:![0-9]+]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !noundef [[META2]]
; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP11]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT:.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -59,30 +93,70 @@ define void @test_5_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
; CHECK-LABEL: define void @test_5_consecutive_loads(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN1:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[TMP15:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
-; CHECK-NEXT: [[TMP16:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load b64, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = freeze b64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b64 [[TMP3]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP4]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i64 [[TMP4]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8
+; CHECK-NEXT: [[TMP10:%.*]] = lshr i64 [[TMP4]], 24
+; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP10]] to i8
+; CHECK-NEXT: [[TMP12:%.*]] = lshr i64 [[TMP4]], 32
+; CHECK-NEXT: [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP9]]
+; CHECK-NEXT: [[SUM3:%.*]] = add i8 [[SUM2]], [[TMP11]]
+; CHECK-NEXT: [[SUM4:%.*]] = add i8 [[SUM3]], [[TMP13]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM4]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 5
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP14]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN1]]
+; CHECK: [[EXIT_PEEL_BEGIN1]]:
+; CHECK-NEXT: [[TMP15:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP16:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 1
; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 2
; CHECK-NEXT: [[P3_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 3
; CHECK-NEXT: [[P4_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 4
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP16]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[D_PEEL:%.*]] = load i8, ptr [[P3_PEEL]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[E_PEEL:%.*]] = load i8, ptr [[P4_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP16]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[D_PEEL:%.*]] = load i8, ptr [[P3_PEEL]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[E_PEEL:%.*]] = load i8, ptr [[P4_PEEL]], align 1, !noundef [[META2]]
; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
; CHECK-NEXT: [[SUM3_PEEL:%.*]] = add i8 [[SUM2_PEEL]], [[D_PEEL]]
; CHECK-NEXT: [[SUM4_PEEL:%.*]] = add i8 [[SUM3_PEEL]], [[E_PEEL]]
; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP15]]
; CHECK-NEXT: store i8 [[SUM4_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP16]], i64 5
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP15]], 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i64 5
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP15]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT:.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -134,14 +208,14 @@ define void @test_8_consecutive_loads_no_peel(ptr %src, ptr %dst, i32 %n) {
; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 5
; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 6
; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 7
-; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[D:%.*]] = load i8, ptr [[P3]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[E:%.*]] = load i8, ptr [[P4]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[F:%.*]] = load i8, ptr [[P5]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[G:%.*]] = load i8, ptr [[P6]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[H:%.*]] = load i8, ptr [[P7]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[D:%.*]] = load i8, ptr [[P3]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[E:%.*]] = load i8, ptr [[P4]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[F:%.*]] = load i8, ptr [[P5]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[G:%.*]] = load i8, ptr [[P6]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[H:%.*]] = load i8, ptr [[P7]], align 1, !noundef [[META2]]
; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
; CHECK-NEXT: [[SUM3:%.*]] = add i8 [[SUM2]], [[D]]
@@ -211,10 +285,10 @@ define void @test_intervening_store(ptr %src, ptr %dst, i32 %n) {
; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
-; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META2]]
; CHECK-NEXT: store i8 42, ptr [[P1]], align 1
-; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META2]]
; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
@@ -259,24 +333,58 @@ define void @test_3_consecutive_i16_loads(ptr %src, ptr %dst, i32 %n) {
; CHECK-LABEL: define void @test_3_consecutive_i16_loads(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN1:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load b64, ptr [[P]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = freeze b64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b64 [[TMP3]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP4]] to i16
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP4]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i16
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i64 [[TMP4]], 32
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i16
+; CHECK-NEXT: [[SUM1:%.*]] = add i16 [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i16 [[SUM1]], [[TMP9]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i16 [[SUM2]], ptr [[DST_I]], align 2
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 6
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP10]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN1]]
+; CHECK: [[EXIT_PEEL_BEGIN1]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i16, ptr [[TMP12]], i64 1
; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i16, ptr [[TMP12]], i64 2
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i16, ptr [[TMP12]], align 2, !noundef [[META0]]
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i16, ptr [[P1_PEEL]], align 2, !noundef [[META0]]
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i16, ptr [[P2_PEEL]], align 2, !noundef [[META0]]
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i16, ptr [[TMP12]], align 2, !noundef [[META2]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i16, ptr [[P1_PEEL]], align 2, !noundef [[META2]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i16, ptr [[P2_PEEL]], align 2, !noundef [[META2]]
; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i16 [[A_PEEL]], [[B_PEEL]]
; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i16 [[SUM1_PEEL]], [[C_PEEL]]
; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP11]]
; CHECK-NEXT: store i16 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 2
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 6
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 6
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP11]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT:.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -311,24 +419,58 @@ define void @test_aa_metadata_preserved(ptr %src, ptr %dst, i32 %n) {
; CHECK-LABEL: define void @test_aa_metadata_preserved(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN1:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load b32, ptr [[P]], align 1, !tbaa [[TBAA5:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = freeze b32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP4]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[TMP4]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP8]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP9]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP10]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN1]]
+; CHECK: [[EXIT_PEEL_BEGIN1]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !tbaa [[TBAA1:![0-9]+]], !noundef [[META0]]
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !tbaa [[TBAA1]], !noundef [[META0]]
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !tbaa [[TBAA1]], !noundef [[META0]]
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !tbaa [[TBAA5]], !noundef [[META2]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !tbaa [[TBAA5]], !noundef [[META2]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !tbaa [[TBAA5]], !noundef [[META2]]
; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP11]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT:.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -364,24 +506,58 @@ define void @test_aa_metadata_different(ptr %src, ptr %dst, i32 %n) {
; CHECK-LABEL: define void @test_aa_metadata_different(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN1:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load b32, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = freeze b32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP4]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[TMP4]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP8]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP9]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP10]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN1]]
+; CHECK: [[EXIT_PEEL_BEGIN1]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !tbaa [[TBAA1]], !noundef [[META0]]
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !tbaa [[TBAA4:![0-9]+]], !noundef [[META0]]
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !tbaa [[TBAA1]], !noundef [[META0]]
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !tbaa [[TBAA5]], !noundef [[META2]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !tbaa [[TBAA10:![0-9]+]], !noundef [[META2]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !tbaa [[TBAA5]], !noundef [[META2]]
; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP11]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT:.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -417,24 +593,59 @@ define void @test_negative_offsets(ptr %src, ptr %dst, i32 %n) {
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SRC_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 2
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = phi ptr [ [[SRC_OFFSET]], %[[ENTRY]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC_OFFSET]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 -2
+; CHECK-NEXT: [[TMP3:%.*]] = load b32, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = freeze b32 [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[TMP4]] to i32
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
+; CHECK-NEXT: [[TMP7:%.*]] = lshr i32 [[TMP5]], 8
+; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP7]] to i8
+; CHECK-NEXT: [[TMP9:%.*]] = lshr i32 [[TMP5]], 16
+; CHECK-NEXT: [[TMP10:%.*]] = trunc i32 [[TMP9]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP10]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP11:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP11]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi ptr [ [[SRC_OFFSET]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
; CHECK-NEXT: [[P_NEG2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 -2
; CHECK-NEXT: [[P_NEG1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 -1
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[P_NEG2_PEEL]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P_NEG1_PEEL]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[TMP13]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[P_NEG2_PEEL]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P_NEG1_PEEL]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[TMP13]], align 1, !noundef [[META2]]
; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP12]]
; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP13]], i64 3
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP12]], 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP12]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT:.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -476,9 +687,9 @@ define void @test_non_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
-; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META2]]
; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
@@ -528,9 +739,9 @@ define void @test_non_inbounds_geps(ptr %src, ptr %dst, i32 %n) {
; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[P]], i64 1
; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[P]], i64 2
-; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META2]]
; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
; CHECK-NEXT: [[DST_I:%.*]] = getelementptr i8, ptr [[DST]], i32 [[I]]
@@ -575,26 +786,62 @@ define void @test_mixed_size_loads(ptr %src, ptr %dst, i32 %n) {
; CHECK-LABEL: define void @test_mixed_size_loads(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN1:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load b32, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = freeze b32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP4]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i16
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[TMP4]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP8]] to i8
+; CHECK-NEXT: [[A_EXT:%.*]] = zext i8 [[TMP5]] to i16
+; CHECK-NEXT: [[SUM1:%.*]] = add i16 [[A_EXT]], [[TMP7]]
+; CHECK-NEXT: [[C_EXT:%.*]] = zext i8 [[TMP9]] to i16
+; CHECK-NEXT: [[SUM2:%.*]] = add i16 [[SUM1]], [[C_EXT]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i16 [[SUM2]], ptr [[DST_I]], align 2
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP10]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN1]]
+; CHECK: [[EXIT_PEEL_BEGIN1]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i16, ptr [[P1_PEEL]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i16, ptr [[P1_PEEL]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !noundef [[META2]]
; CHECK-NEXT: [[A_EXT_PEEL:%.*]] = zext i8 [[A_PEEL]] to i16
; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i16 [[A_EXT_PEEL]], [[B_PEEL]]
; CHECK-NEXT: [[C_EXT_PEEL:%.*]] = zext i8 [[C_PEEL]] to i16
; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i16 [[SUM1_PEEL]], [[C_EXT_PEEL]]
; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP11]]
; CHECK-NEXT: store i16 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 2
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP11]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT:.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -689,9 +936,9 @@ define void @test_step_mismatch(ptr %src, ptr %dst, i32 %n) {
; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
-; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1, !noundef [[META2]]
; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
@@ -735,24 +982,58 @@ define void @test_alignment_inferred_from_group(ptr %src, ptr %dst, i32 %n) {
; CHECK-LABEL: define void @test_alignment_inferred_from_group(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN1:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load b32, ptr [[P]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = freeze b32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP4]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[TMP4]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP8]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP9]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP10]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN1]]
+; CHECK: [[EXIT_PEEL_BEGIN1]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 4, !noundef [[META0]]
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 4, !noundef [[META2]]
; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP11]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT:.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -788,10 +1069,45 @@ define void @test_loads_nonpoison_via_address_use(ptr %in, ptr %weights, ptr %ou
; CHECK-LABEL: define void @test_loads_nonpoison_via_address_use(
; CHECK-SAME: ptr [[IN:%.*]], ptr [[WEIGHTS:%.*]], ptr [[OUT:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN1:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[IN]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[IN]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load b32, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = freeze b32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP4]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[TMP4]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP8]] to i8
+; CHECK-NEXT: [[R32:%.*]] = zext i8 [[TMP5]] to i32
+; CHECK-NEXT: [[G32:%.*]] = zext i8 [[TMP7]] to i32
+; CHECK-NEXT: [[B32:%.*]] = zext i8 [[TMP9]] to i32
+; CHECK-NEXT: [[S1:%.*]] = add i32 [[R32]], [[G32]]
+; CHECK-NEXT: [[IDX:%.*]] = add i32 [[S1]], [[B32]]
+; CHECK-NEXT: [[W_PTR:%.*]] = getelementptr inbounds i8, ptr [[WEIGHTS]], i32 [[IDX]]
+; CHECK-NEXT: [[W:%.*]] = load i8, ptr [[W_PTR]], align 1
+; CHECK-NEXT: [[OUT_I:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i32 [[I]]
+; CHECK-NEXT: store i8 [[W]], ptr [[OUT_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP10]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN1]]
+; CHECK: [[EXIT_PEEL_BEGIN1]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[IN]], %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
; CHECK-NEXT: [[G_PTR_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
; CHECK-NEXT: [[B_PTR_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
; CHECK-NEXT: [[R_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1
@@ -806,11 +1122,15 @@ define void @test_loads_nonpoison_via_address_use(ptr %in, ptr %weights, ptr %ou
; CHECK-NEXT: [[W_PEEL:%.*]] = load i8, ptr [[W_PTR_PEEL]], align 1
; CHECK-NEXT: [[OUT_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i32 [[TMP11]]
; CHECK-NEXT: store i8 [[W_PEEL]], ptr [[OUT_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP11]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT:.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -953,10 +1273,41 @@ define void @test_call_willreturn(ptr %src, ptr %dst, i32 %n) {
; CHECK-LABEL: define void @test_call_willreturn(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN1:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load b32, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = freeze b32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP4]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[TMP4]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP8]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP9]]
+; CHECK-NEXT: call void @sink_willreturn(i8 [[SUM2]])
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP10]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN1]]
+; CHECK: [[EXIT_PEEL_BEGIN1]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1
@@ -967,11 +1318,15 @@ define void @test_call_willreturn(ptr %src, ptr %dst, i32 %n) {
; CHECK-NEXT: call void @sink_willreturn(i8 [[SUM2_PEEL]])
; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP11]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT:.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -1071,8 +1426,8 @@ define void @test_offset_exceeds_int64(ptr addrspace(2) %src, ptr %dst, i32 %n)
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[P:%.*]] = phi ptr addrspace(2) [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[PBIG:%.*]] = getelementptr inbounds i8, ptr addrspace(2) [[P]], i128 100000000000000000000
-; CHECK-NEXT: [[A:%.*]] = load i8, ptr addrspace(2) [[P]], align 1, !noundef [[META0]]
-; CHECK-NEXT: [[B:%.*]] = load i8, ptr addrspace(2) [[PBIG]], align 1, !noundef [[META0]]
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr addrspace(2) [[P]], align 1, !noundef [[META2]]
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr addrspace(2) [[PBIG]], align 1, !noundef [[META2]]
; CHECK-NEXT: [[SUM:%.*]] = add i8 [[A]], [[B]]
; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
; CHECK-NEXT: store i8 [[SUM]], ptr [[DST_I]], align 1
@@ -1133,7 +1488,7 @@ define void @test_peel_last_for_compare_not_widened(ptr %src, i32 %n) {
; CHECK-NEXT: [[IV_NEXT]] = add nuw i32 [[IV]], 1
; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[N]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp ne i32 [[IV_NEXT]], [[TMP1]]
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[IV_NEXT]], %[[LOOP]] ]
; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
@@ -1198,21 +1553,52 @@ define void @test_i24_loads(ptr %src, ptr %dst, i32 %n) {
; CHECK-LABEL: define void @test_i24_loads(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN1:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load b64, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = freeze b64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b64 [[TMP3]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP4]] to i24
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP4]], 24
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i24
+; CHECK-NEXT: [[SUM:%.*]] = add i24 [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i24, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i24 [[SUM]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 6
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP8]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN1]]
+; CHECK: [[EXIT_PEEL_BEGIN1]]:
+; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 3
; CHECK-NEXT: [[A_PEEL:%.*]] = load i24, ptr [[TMP10]], align 1
; CHECK-NEXT: [[B_PEEL:%.*]] = load i24, ptr [[P1_PEEL]], align 1
; CHECK-NEXT: [[SUM_PEEL:%.*]] = add i24 [[A_PEEL]], [[B_PEEL]]
; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i24, ptr [[DST]], i32 [[TMP9]]
; CHECK-NEXT: store i24 [[SUM_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP10]], i64 6
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 6
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP9]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT:.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -1241,11 +1627,41 @@ define void @test_base_plus_iv_pattern(ptr %src, ptr %dst, i32 %n) {
; CHECK-LABEL: define void @test_base_plus_iv_pattern(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN1:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP_PEEL]] ]
; CHECK-NEXT: [[I_EXT_PEEL:%.*]] = zext i32 [[TMP11]] to i64
-; CHECK-NEXT: [[STRIDE_PEEL:%.*]] = mul i64 [[I_EXT_PEEL]], 3
+; CHECK-NEXT: [[STRIDE:%.*]] = mul nuw nsw i64 [[I_EXT_PEEL]], 3
+; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[STRIDE]]
+; CHECK-NEXT: [[TMP2:%.*]] = load b32, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = freeze b32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP4]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[TMP4]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP8]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP9]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[TMP11]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP10]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN1]]
+; CHECK: [[EXIT_PEEL_BEGIN1]]:
+; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
+; CHECK-NEXT: [[I_EXT_PEEL1:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT: [[STRIDE_PEEL:%.*]] = mul i64 [[I_EXT_PEEL1]], 3
; CHECK-NEXT: [[P_PEEL:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[STRIDE_PEEL]]
; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[P_PEEL]], i64 1
; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[P_PEEL]], i64 2
@@ -1254,12 +1670,16 @@ define void @test_base_plus_iv_pattern(ptr %src, ptr %dst, i32 %n) {
; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
-; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP12]]
; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP12]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT:.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -1292,11 +1712,28 @@ define void @test_conditional_group_not_widened(ptr %pa, ptr %pb, ptr %dst, i32
; CHECK-LABEL: define void @test_conditional_group_not_widened(
; CHECK-SAME: ptr [[PA:%.*]], ptr [[PB:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]], i1 [[C0:%.*]]) {
; CHECK-NEXT: [[EXIT_PEEL_BEGIN1:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
; CHECK: [[LOOP_PEEL1]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN1]] ], [ [[I_NEXT_PEEL:%.*]], %[[LATCH_PEEL:.*]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = phi ptr [ [[PA]], %[[EXIT_PEEL_BEGIN1]] ], [ [[A_NEXT_PEEL:%.*]], %[[LATCH_PEEL]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = phi ptr [ [[PB]], %[[EXIT_PEEL_BEGIN1]] ], [ [[B_NEXT_PEEL:%.*]], %[[LATCH_PEEL]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[A:%.*]] = phi ptr [ [[PA]], %[[ENTRY_SPLIT]] ], [ [[A_NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi ptr [ [[PB]], %[[ENTRY_SPLIT]] ], [ [[B_NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load b32, ptr [[A]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = freeze b32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP4]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[TMP4]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP8]] to i8
+; CHECK-NEXT: [[AS1:%.*]] = add i8 [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[AS2:%.*]] = add i8 [[AS1]], [[TMP9]]
+; CHECK-NEXT: store i8 [[AS2]], ptr [[DST]], align 1
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN:.*]], label %[[LATCH]]
+; CHECK: [[THEN]]:
; CHECK-NEXT: [[A1_PEEL1:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 1
; CHECK-NEXT: [[A2_PEEL1:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 2
; CHECK-NEXT: [[AV0_PEEL1:%.*]] = load i8, ptr [[TMP14]], align 1
@@ -1304,8 +1741,36 @@ define void @test_conditional_group_not_widened(ptr %pa, ptr %pb, ptr %dst, i32
; CHECK-NEXT: [[AV2_PEEL1:%.*]] = load i8, ptr [[A2_PEEL1]], align 1
; CHECK-NEXT: [[AS1_PEEL1:%.*]] = add i8 [[AV0_PEEL1]], [[AV1_PEEL1]]
; CHECK-NEXT: [[AS2_PEEL1:%.*]] = add i8 [[AS1_PEEL1]], [[AV2_PEEL1]]
-; CHECK-NEXT: store i8 [[AS2_PEEL1]], ptr [[DST]], align 1
-; CHECK-NEXT: br i1 [[C0]], label %[[THEN_PEEL:.*]], label %[[LATCH_PEEL]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[AS2_PEEL1]], ptr [[DST_I]], align 1
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[A_NEXT]] = getelementptr inbounds i8, ptr [[A]], i64 3
+; CHECK-NEXT: [[B_NEXT]] = getelementptr inbounds i8, ptr [[TMP14]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP10]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL1]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LATCH]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[A_NEXT]], %[[LATCH]] ]
+; CHECK-NEXT: [[DOTPH2:%.*]] = phi ptr [ [[B_NEXT]], %[[LATCH]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN1]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[PA]], %[[EXIT_PEEL_BEGIN1]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi ptr [ [[PB]], %[[EXIT_PEEL_BEGIN1]] ], [ [[DOTPH2]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[A1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 1
+; CHECK-NEXT: [[A2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
+; CHECK-NEXT: [[AV0_PEEL:%.*]] = load i8, ptr [[TMP12]], align 1
+; CHECK-NEXT: [[AV1_PEEL:%.*]] = load i8, ptr [[A1_PEEL]], align 1
+; CHECK-NEXT: [[AV2_PEEL:%.*]] = load i8, ptr [[A2_PEEL]], align 1
+; CHECK-NEXT: [[AS1_PEEL:%.*]] = add i8 [[AV0_PEEL]], [[AV1_PEEL]]
+; CHECK-NEXT: [[AS2_PEEL:%.*]] = add i8 [[AS1_PEEL]], [[AV2_PEEL]]
+; CHECK-NEXT: store i8 [[AS2_PEEL]], ptr [[DST]], align 1
+; CHECK-NEXT: br i1 [[C0]], label %[[THEN_PEEL:.*]], label %[[LATCH_PEEL:.*]]
; CHECK: [[THEN_PEEL]]:
; CHECK-NEXT: [[B1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 1
; CHECK-NEXT: [[B2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 2
@@ -1318,12 +1783,16 @@ define void @test_conditional_group_not_widened(ptr %pa, ptr %pb, ptr %dst, i32
; CHECK-NEXT: store i8 [[BS2_PEEL]], ptr [[DST_I_PEEL]], align 1
; CHECK-NEXT: br label %[[LATCH_PEEL]]
; CHECK: [[LATCH_PEEL]]:
-; CHECK-NEXT: [[A_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP14]], i64 3
-; CHECK-NEXT: [[B_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP13]], i64 3
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[A_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[B_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP11]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL1]], label %[[EXIT1:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT1:.*]], label %[[EXIT1]]
; CHECK: [[EXIT1]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
@@ -1380,10 +1849,44 @@ define void @test_unordered_loads_use_between(ptr %src, ptr %dst, i32 %n) {
; CHECK-LABEL: define void @test_unordered_loads_use_between(
; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[EXIT_PEEL_BEGIN:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN1:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load b32, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = freeze b32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP4]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i32 [[TMP4]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP8]] to i8
+; CHECK-NEXT: [[CZ:%.*]] = zext i8 [[TMP9]] to i32
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP9]]
+; CHECK-NEXT: [[SZ:%.*]] = zext i8 [[SUM2]] to i32
+; CHECK-NEXT: [[TOT:%.*]] = add i32 [[CZ]], [[SZ]]
+; CHECK-NEXT: [[T:%.*]] = trunc i32 [[TOT]] to i8
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[T]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP10]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP_PEEL]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP_PEEL]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN1]]
+; CHECK: [[EXIT_PEEL_BEGIN1]]:
+; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi ptr [ [[SRC]], %[[EXIT_PEEL_BEGIN]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL1:.*]]
+; CHECK: [[LOOP_PEEL1]]:
; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 2
; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
; CHECK-NEXT: [[CZ_PEEL:%.*]] = zext i8 [[C_PEEL]] to i32
@@ -1397,11 +1900,15 @@ define void @test_unordered_loads_use_between(ptr %src, ptr %dst, i32 %n) {
; CHECK-NEXT: [[T_PEEL:%.*]] = trunc i32 [[TOT_PEEL]] to i8
; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP11]]
; CHECK-NEXT: store i8 [[T_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP11]], 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP11]], 1
; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP_PEEL]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT:.*]], label %[[EXIT]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -1706,12 +2213,26 @@ exit:
attributes #0 = { "target-features"="+strict-align" }
;.
-; CHECK: [[META0]] = !{}
-; CHECK: [[TBAA1]] = !{[[META2:![0-9]+]], [[META2]], i64 0}
-; CHECK: [[META2]] = !{!"char", [[META3:![0-9]+]], i64 0}
-; CHECK: [[META3]] = !{!"root"}
-; CHECK: [[TBAA4]] = !{[[META5:![0-9]+]], [[META5]], i64 0}
-; CHECK: [[META5]] = !{!"int", [[META3]], i64 0}
-; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META7:![0-9]+]]}
-; CHECK: [[META7]] = !{!"llvm.loop.peeled.count", i32 1}
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
+; CHECK: [[META2]] = !{}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]]}
+; CHECK: [[TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0}
+; CHECK: [[META6]] = !{!"char", [[META7:![0-9]+]], i64 0}
+; CHECK: [[META7]] = !{!"root"}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}
+; CHECK: [[TBAA10]] = !{[[META11:![0-9]+]], [[META11]], i64 0}
+; CHECK: [[META11]] = !{!"int", [[META7]], i64 0}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]]}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]]}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]]}
+; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META1]]}
+; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META1]]}
+; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META1]]}
;.
More information about the llvm-commits
mailing list