[llvm] [AggressiveInstCombine] Fold split-width i32 cttz/ctlz patterns into wide i64 intrinsics (PR #192296)
Usha Gupta via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 04:58:37 PDT 2026
https://github.com/usha1830 updated https://github.com/llvm/llvm-project/pull/192296
>From 29a700e28572dcd70d1a8a931f707479689b66bd Mon Sep 17 00:00:00 2001
From: Usha Gupta <usha.gupta at arm.com>
Date: Wed, 15 Apr 2026 16:22:22 +0000
Subject: [PATCH 1/3] Pre-commit tests for folding split-width i32 cttz/ctlz
into wide (i64) intrinsics
---
.../AggressiveInstCombine/fold-split-ctlz.ll | 252 +++++++++++++++++
.../AggressiveInstCombine/fold-split-cttz.ll | 255 ++++++++++++++++++
2 files changed, 507 insertions(+)
create mode 100644 llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
create mode 100644 llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
new file mode 100644
index 0000000000000..3015d9ece004c
--- /dev/null
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
@@ -0,0 +1,252 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=aggressive-instcombine -S | FileCheck %s
+
+; Phi-based split i64 ctlz: branch on whether upper 32 bits are zero,
+; ctlz each half separately, merge via phi.
+define i32 @split_ctlz_phi(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_phi(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc nuw i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i32 [[TMP0]], 32
+; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 63, i32 [[TMP2]]
+; CHECK-NEXT: br label [[CLEANUP:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV]], i1 true)
+; CHECK-NEXT: br label [[CLEANUP]]
+; CHECK: cleanup:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %cmp = icmp eq i64 %shr, 0
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+ %conv1 = trunc nuw i64 %val to i32
+ %0 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv1, i1 true)
+ %1 = icmp eq i64 %val, 0
+ %2 = or disjoint i32 %0, 32
+ %add = select i1 %1, i32 63, i32 %2
+ br label %cleanup
+
+if.else:
+ %conv = trunc nuw i64 %shr to i32
+ %3 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv, i1 true)
+ br label %cleanup
+
+cleanup:
+ %retval.0 = phi i32 [ %add, %if.then ], [ %3, %if.else ]
+ ret i32 %retval.0
+}
+
+; Same pattern but with add instead of or.
+define i32 @split_ctlz_phi_add(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_phi_add(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc nuw i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV1]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP0]], 32
+; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 63, i32 [[TMP2]]
+; CHECK-NEXT: br label [[CLEANUP:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV]], i1 true)
+; CHECK-NEXT: br label [[CLEANUP]]
+; CHECK: cleanup:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %cmp = icmp eq i64 %shr, 0
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+ %conv1 = trunc nuw i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv1, i1 false)
+ %1 = icmp eq i64 %val, 0
+ %2 = add i32 %0, 32
+ %add = select i1 %1, i32 63, i32 %2
+ br label %cleanup
+
+if.else:
+ %conv = trunc nuw i64 %shr to i32
+ %3 = call i32 @llvm.ctlz.i32(i32 %conv, i1 true)
+ br label %cleanup
+
+cleanup:
+ %retval.0 = phi i32 [ %add, %if.then ], [ %3, %if.else ]
+ ret i32 %retval.0
+}
+
+; Select-based split i64 ctlz
+define i32 @split_ctlz_select(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP2]]
+; CHECK-NEXT: ret i32 [[TMP1]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %conv_hi = trunc nuw i64 %shr to i32
+ %cmp = icmp eq i32 %conv_hi, 0
+ %conv_lo = trunc nuw i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Select-based with i64 comparison on lshr result (not truncated).
+define i32 @split_ctlz_select_i64cmp(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_i64cmp(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP2]]
+; CHECK-NEXT: ret i32 [[TMP1]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %cmp = icmp eq i64 %shr, 0
+ %conv_hi = trunc nuw i64 %shr to i32
+ %conv_lo = trunc nuw i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Select-based with icmp ne (inverted condition).
+define i32 @split_ctlz_select_ne(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_ne(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[CONV_HI]], 0
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[TMP2]], i32 [[ADD]]
+; CHECK-NEXT: ret i32 [[TMP1]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %conv_hi = trunc nuw i64 %shr to i32
+ %cmp = icmp ne i32 %conv_hi, 0
+ %conv_lo = trunc nuw i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %1, i32 %add
+ ret i32 %retval
+}
+
+; Select-based with or instead of add.
+define i32 @split_ctlz_select_or(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_or(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = or disjoint i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP2]]
+; CHECK-NEXT: ret i32 [[TMP1]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %conv_hi = trunc nuw i64 %shr to i32
+ %cmp = icmp eq i32 %conv_hi, 0
+ %conv_lo = trunc nuw i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = or disjoint i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Negative test: mismatched source values, should NOT be folded.
+define i32 @split_ctlz_different_sources(i64 noundef %val, i64 noundef %val2) {
+; CHECK-LABEL: @split_ctlz_different_sources(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL2:%.*]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %conv_hi = trunc nuw i64 %shr to i32
+ %cmp = icmp eq i32 %conv_hi, 0
+ %conv_lo = trunc nuw i64 %val2 to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Negative test: wrong shift amount, should NOT be folded.
+define i32 @split_ctlz_wrong_shift(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_wrong_shift(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 16
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc i64 [[SHR]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %shr = lshr i64 %val, 16
+ %conv_hi = trunc i64 %shr to i32
+ %cmp = icmp eq i32 %conv_hi, 0
+ %conv_lo = trunc i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+declare i32 @llvm.ctlz.i32(i32, i1 immarg)
+declare i64 @llvm.ctlz.i64(i64, i1 immarg)
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
new file mode 100644
index 0000000000000..45212cd53dc96
--- /dev/null
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
@@ -0,0 +1,255 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=aggressive-instcombine -S | FileCheck %s
+
+; Phi-based split i64 cttz: branch on whether low 32 bits are zero,
+; cttz each half separately, merge via phi.
+define i32 @split_cttz_phi(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_phi(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[SHR:%.*]] = lshr exact i64 [[VAL]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i32 [[TMP0]], 32
+; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 32, i32 [[TMP2]]
+; CHECK-NEXT: br label [[CLEANUP:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP4:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: br label [[CLEANUP]]
+; CHECK: cleanup:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+ %shr = lshr exact i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv, i1 true)
+ %1 = icmp eq i64 %val, 0
+ %2 = or disjoint i32 %0, 32
+ %add = select i1 %1, i32 32, i32 %2
+ br label %cleanup
+
+if.else:
+ %3 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ br label %cleanup
+
+cleanup:
+ %retval.0 = phi i32 [ %add, %if.then ], [ %3, %if.else ]
+ ret i32 %retval.0
+}
+
+; Same pattern but with add instead of or for the +32.
+define i32 @split_cttz_phi_add(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_phi_add(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[SHR:%.*]] = lshr exact i64 [[VAL]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP0]], 32
+; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 32, i32 [[TMP2]]
+; CHECK-NEXT: br label [[CLEANUP:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: br label [[CLEANUP]]
+; CHECK: cleanup:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+ %shr = lshr exact i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %1 = icmp eq i64 %val, 0
+ %2 = add i32 %0, 32
+ %add = select i1 %1, i32 32, i32 %2
+ br label %cleanup
+
+if.else:
+ %3 = call i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ br label %cleanup
+
+cleanup:
+ %retval.0 = phi i32 [ %add, %if.then ], [ %3, %if.else ]
+ ret i32 %retval.0
+}
+
+; Select-based split i64 cttz: both cttz.i32 are speculated,
+; result picked by select.
+define i32 @split_cttz_select(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_select(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP2]], 32
+; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP3]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ %shr = lshr i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Select-based with or instead of add.
+define i32 @split_cttz_select_or(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_or(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = or disjoint i32 [[TMP2]], 32
+; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP3]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ %shr = lshr i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = or disjoint i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Select-based with icmp ne (inverted condition).
+define i32 @split_cttz_select_ne(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_ne(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP2]], 32
+; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[TMP3]], i32 [[ADD]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp ne i32 %conv1, 0
+ %shr = lshr i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %1, i32 %add
+ ret i32 %retval
+}
+
+; Select-based cttz used inside a larger expression (>> 3)
+define i32 @split_cttz_select_shift(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_shift(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP2]], 32
+; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP3]]
+; CHECK-NEXT: [[RESULT:%.*]] = lshr i32 [[RETVAL]], 3
+; CHECK-NEXT: ret i32 [[RESULT]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ %shr = lshr i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ %result = lshr i32 %retval, 3
+ ret i32 %result
+}
+
+; Negative test: mismatched source values, should NOT be folded.
+define i32 @split_cttz_different_sources(i64 noundef %val, i64 noundef %val2) {
+; CHECK-LABEL: @split_cttz_different_sources(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL2:%.*]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ %shr = lshr i64 %val2, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Negative test: shift amount is not 32, should NOT be folded.
+define i32 @split_cttz_wrong_shift(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_wrong_shift(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL]], 16
+; CHECK-NEXT: [[CONV:%.*]] = trunc i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ %shr = lshr i64 %val, 16
+ %conv = trunc i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+declare i32 @llvm.cttz.i32(i32, i1 immarg)
+declare i64 @llvm.cttz.i64(i64, i1 immarg)
>From a1cd7e88caa9abb74875b5256e0645ade75931a8 Mon Sep 17 00:00:00 2001
From: Usha Gupta <usha.gupta at arm.com>
Date: Wed, 15 Apr 2026 16:42:11 +0000
Subject: [PATCH 2/3] Fold split-width (i32) cttz/ctlz patterns into wide (i64)
intrinsics
---
.../AggressiveInstCombine.cpp | 445 ++++++++++++++++++
.../AggressiveInstCombine/fold-split-ctlz.ll | 64 +--
.../AggressiveInstCombine/fold-split-cttz.ll | 72 +--
3 files changed, 481 insertions(+), 100 deletions(-)
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index 7e57c5eff0bf1..bb5cacf43ad9f 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -53,6 +53,14 @@ STATISTIC(NumGuardedRotates,
STATISTIC(NumGuardedFunnelShifts,
"Number of guarded funnel shifts transformed into funnel shifts");
STATISTIC(NumPopCountRecognized, "Number of popcount idioms recognized");
+STATISTIC(NumSplitCTTZFolded,
+ "Number of split-width cttz patterns folded into full-width cttz");
+STATISTIC(NumSplitCTLZFolded,
+ "Number of split-width ctlz patterns folded into full-width ctlz");
+STATISTIC(NumSelectCTTZFolded,
+ "Number of select-based split cttz patterns folded");
+STATISTIC(NumSelectCTLZFolded,
+ "Number of select-based split ctlz patterns folded");
static cl::opt<unsigned> MaxInstrsToScan(
"aggressive-instcombine-max-scan-instrs", cl::init(64), cl::Hidden,
@@ -68,6 +76,439 @@ static cl::opt<unsigned>
cl::desc("The maximum length of a constant string to "
"inline a memchr call."));
+/// Try to fold a phi that merges two 32-bit cttz operations into a single
+/// 64-bit cttz. This pattern arises when a 64-bit cttz is implemented as
+/// two 32-bit cttz operations with a branch on whether the low half is zero:
+///
+/// GuardBB:
+/// %lo = trunc i64 %val to i32
+/// %cmp = icmp eq i32 %lo, 0
+/// br i1 %cmp, label %HiBB, label %LoBB
+///
+/// HiBB:
+/// %shr = lshr i64 %val, 32
+/// %hi = trunc i64 %shr to i32
+/// %cttz_hi = call i32 @llvm.cttz.i32(i32 %hi, i1 false)
+/// %val_is_zero = icmp eq i64 %val, 0
+/// %cttz_hi_plus32 = or i32 %cttz_hi, 32
+/// %hi_result = select i1 %val_is_zero, i32 ZeroResult, i32 %cttz_hi_plus32
+/// br label %PhiBB
+///
+/// LoBB:
+/// %cttz_lo = call i32 @llvm.cttz.i32(i32 %lo, i1 true)
+/// br label %PhiBB
+///
+/// PhiBB:
+/// %result = phi i32 [%hi_result, %HiBB], [%cttz_lo, %LoBB]
+/// -->
+/// %cttz64 = call i64 @llvm.cttz.i64(i64 %val, i1 true)
+/// %trunc = trunc i64 %cttz64 to i32
+/// %val_is_zero = icmp eq i64 %val, 0
+/// %result = select i1 %val_is_zero, i32 ZeroResult, i32 %trunc
+///
+/// Alive proof: https://alive2.llvm.org/ce/z/FHgJpq
+static bool foldSplitWidthCTTZ(Instruction &I, const DominatorTree &DT) {
+ if (I.getOpcode() != Instruction::PHI || I.getNumOperands() != 2)
+ return false;
+
+ if (!I.getType()->isIntegerTy(32))
+ return false;
+
+ PHINode &Phi = cast<PHINode>(I);
+
+ // Try both orderings of phi operands.
+ for (unsigned HiIdx = 0; HiIdx < 2; ++HiIdx) {
+ unsigned LoIdx = 1 - HiIdx;
+ Value *HiVal = Phi.getIncomingValue(HiIdx);
+ Value *LoVal = Phi.getIncomingValue(LoIdx);
+ BasicBlock *HiBB = Phi.getIncomingBlock(HiIdx);
+ BasicBlock *LoBB = Phi.getIncomingBlock(LoIdx);
+
+ // Match LoVal: cttz.i32(trunc(SrcVal to i32), _)
+ Value *LoTrunc;
+ if (!match(LoVal,
+ m_Intrinsic<Intrinsic::cttz>(m_Value(LoTrunc), m_Value())))
+ continue;
+
+ // LoTrunc must be a trunc of an i64 value.
+ Value *SrcVal = nullptr;
+ if (!match(LoTrunc, m_Trunc(m_Value(SrcVal))))
+ continue;
+ if (!SrcVal->getType()->isIntegerTy(64))
+ continue;
+
+ // Match HiVal: select(icmp eq SrcVal, 0, C,
+ // or(cttz.i32(trunc(lshr(SrcVal, 32)), _), 32))
+ // where C is the result when the full value is zero.
+ Value *SelectCond, *SelectTrue, *SelectFalse;
+ if (!match(HiVal, m_Select(m_Value(SelectCond), m_Value(SelectTrue),
+ m_Value(SelectFalse))))
+ continue;
+
+ // SelectCond: icmp eq SrcVal, 0
+ if (!match(SelectCond, m_SpecificICmp(CmpInst::ICMP_EQ, m_Specific(SrcVal),
+ m_ZeroInt())))
+ continue;
+
+ // SelectTrue: the zero-input result
+ const APInt *ZeroResult;
+ if (!match(SelectTrue, m_APInt(ZeroResult)))
+ continue;
+
+ // SelectFalse: or(cttz.i32(trunc(lshr(SrcVal, 32)), _), 32)
+ // Also accept add instead of or.
+ Value *CttzHiCall;
+ if (!match(SelectFalse, m_c_Or(m_Value(CttzHiCall), m_SpecificInt(32))) &&
+ !match(SelectFalse, m_c_Add(m_Value(CttzHiCall), m_SpecificInt(32))))
+ continue;
+
+ // CttzHiCall: cttz.i32(trunc(lshr(SrcVal, 32)), _)
+ Value *HiTrunc;
+ if (!match(CttzHiCall,
+ m_Intrinsic<Intrinsic::cttz>(m_Value(HiTrunc), m_Value())))
+ continue;
+
+ if (!match(HiTrunc, m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(32)))))
+ continue;
+
+ // Both HiBB and LoBB share a single predecessor
+ // that branches on whether the low 32 bits are zero.
+ BasicBlock *GuardBB = LoBB->getSinglePredecessor();
+ if (!GuardBB || GuardBB != HiBB->getSinglePredecessor())
+ continue;
+
+ Instruction *GuardTerm = GuardBB->getTerminator();
+ // Guard: br (icmp eq (trunc SrcVal to i32), 0), HiBB, LoBB
+ if (!match(GuardTerm,
+ m_Br(m_SpecificICmp(CmpInst::ICMP_EQ,
+ m_Trunc(m_Specific(SrcVal)), m_ZeroInt()),
+ m_SpecificBB(HiBB), m_SpecificBB(LoBB))))
+ continue;
+
+ // Verify SrcVal dominates the guard.
+ if (auto *SrcI = dyn_cast<Instruction>(SrcVal))
+ if (!DT.dominates(SrcI, GuardTerm))
+ continue;
+
+ // Match successful.
+ BasicBlock *PhiBB = Phi.getParent();
+ IRBuilder<> Builder(PhiBB, PhiBB->getFirstInsertionPt());
+
+ Value *Cttz64 = Builder.CreateIntrinsic(
+ Intrinsic::cttz, {SrcVal->getType()}, {SrcVal, Builder.getTrue()});
+ Value *Trunc = Builder.CreateTrunc(Cttz64, Builder.getInt32Ty());
+ Value *IsZero =
+ Builder.CreateICmpEQ(SrcVal, ConstantInt::get(SrcVal->getType(), 0));
+ Value *Result = Builder.CreateSelect(
+ IsZero, ConstantInt::get(Builder.getInt32Ty(), *ZeroResult), Trunc);
+
+ Phi.replaceAllUsesWith(Result);
+ ++NumSplitCTTZFolded;
+ return true;
+ }
+
+ return false;
+}
+
+/// Try to fold a phi that merges two 32-bit ctlz operations into a single
+/// 64-bit ctlz. This is similar to foldSplitWidthCTTZ, but for leading zeros.
+///
+/// GuardBB:
+/// %shr = lshr i64 %val, 32
+/// %cmp = icmp eq i64 %shr, 0
+/// br i1 %cmp, label %LoBB, label %HiBB
+///
+/// LoBB (upper is zero: count in lower half + 32):
+/// %lo = trunc i64 %val to i32
+/// %ctlz_lo = call i32 @llvm.ctlz.i32(i32 %lo, i1 true)
+/// %val_is_zero = icmp eq i64 %val, 0
+/// %ctlz_lo_plus32 = or i32 %ctlz_lo, 32
+/// %lo_result = select i1 %val_is_zero, i32 ZeroResult, i32 %ctlz_lo_plus32
+/// br label %PhiBB
+///
+/// HiBB (upper is non-zero: count in upper half):
+/// %hi = trunc i64 %shr to i32
+/// %ctlz_hi = call i32 @llvm.ctlz.i32(i32 %hi, i1 true)
+/// br label %PhiBB
+///
+/// PhiBB:
+/// %result = phi i32 [%lo_result, %LoBB], [%ctlz_hi, %HiBB]
+/// -->
+/// %ctlz64 = call i64 @llvm.ctlz.i64(i64 %val, i1 true)
+/// %trunc = trunc i64 %ctlz64 to i32
+/// %val_is_zero = icmp eq i64 %val, 0
+/// %result = select i1 %val_is_zero, i32 ZeroResult, i32 %trunc
+///
+/// Alive proof: https://alive2.llvm.org/ce/z/peel2Y
+static bool foldSplitWidthCTLZ(Instruction &I, const DominatorTree &DT) {
+ if (I.getOpcode() != Instruction::PHI || I.getNumOperands() != 2)
+ return false;
+
+ if (!I.getType()->isIntegerTy(32))
+ return false;
+
+ PHINode &Phi = cast<PHINode>(I);
+
+ // Try both orderings of phi operands.
+ for (unsigned HiIdx = 0; HiIdx < 2; ++HiIdx) {
+ unsigned LoIdx = 1 - HiIdx;
+ Value *LoVal = Phi.getIncomingValue(LoIdx);
+ Value *HiVal = Phi.getIncomingValue(HiIdx);
+ BasicBlock *LoBB = Phi.getIncomingBlock(LoIdx);
+ BasicBlock *HiBB = Phi.getIncomingBlock(HiIdx);
+
+ // Match HiVal: ctlz.i32(trunc(lshr(SrcVal, 32) to i32), _)
+ // This is the ctlz on the upper 32 bits (when upper is non-zero).
+ Value *HiTrunc;
+ if (!match(HiVal,
+ m_Intrinsic<Intrinsic::ctlz>(m_Value(HiTrunc), m_Value())))
+ continue;
+
+ // HiTrunc must be trunc(lshr(SrcVal, 32)).
+ Value *SrcVal = nullptr;
+ if (!match(HiTrunc, m_Trunc(m_LShr(m_Value(SrcVal), m_SpecificInt(32)))))
+ continue;
+ if (!SrcVal->getType()->isIntegerTy(64))
+ continue;
+
+ // Match LoVal: select(icmp eq SrcVal, 0, ZeroResult,
+ // or/add(ctlz.i32(trunc(SrcVal to i32), _), 32))
+ Value *SelectCond, *SelectTrue, *SelectFalse;
+ if (!match(LoVal, m_Select(m_Value(SelectCond), m_Value(SelectTrue),
+ m_Value(SelectFalse))))
+ continue;
+
+ // SelectCond: icmp eq SrcVal, 0
+ if (!match(SelectCond, m_SpecificICmp(CmpInst::ICMP_EQ, m_Specific(SrcVal),
+ m_ZeroInt())))
+ continue;
+
+ // SelectTrue: the zero-input result.
+ const APInt *ZeroResult;
+ if (!match(SelectTrue, m_APInt(ZeroResult)))
+ continue;
+
+ // SelectFalse: or/add(ctlz.i32(trunc(SrcVal to i32), _), 32)
+ Value *CtlzLoCall;
+ if (!match(SelectFalse, m_c_Or(m_Value(CtlzLoCall), m_SpecificInt(32))) &&
+ !match(SelectFalse, m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(32))))
+ continue;
+
+ // CtlzLoCall: ctlz.i32(trunc(SrcVal to i32), _)
+ Value *LoTrunc;
+ if (!match(CtlzLoCall,
+ m_Intrinsic<Intrinsic::ctlz>(m_Value(LoTrunc), m_Value())))
+ continue;
+
+ if (!match(LoTrunc, m_Trunc(m_Specific(SrcVal))))
+ continue;
+
+ // Both LoBB and HiBB share a single predecessor that branches on whether
+ // the upper 32 bits are zero.
+ BasicBlock *GuardBB = LoBB->getSinglePredecessor();
+ if (!GuardBB || GuardBB != HiBB->getSinglePredecessor())
+ continue;
+
+ Instruction *GuardTerm = GuardBB->getTerminator();
+ // Guard: br (icmp eq (lshr SrcVal, 32), 0), LoBB, HiBB
+ if (!match(GuardTerm, m_Br(m_SpecificICmp(CmpInst::ICMP_EQ,
+ m_LShr(m_Specific(SrcVal),
+ m_SpecificInt(32)),
+ m_ZeroInt()),
+ m_SpecificBB(LoBB), m_SpecificBB(HiBB))))
+ continue;
+
+ // Verify SrcVal dominates the guard.
+ if (auto *SrcI = dyn_cast<Instruction>(SrcVal))
+ if (!DT.dominates(SrcI, GuardTerm))
+ continue;
+
+ // Match successful.
+ BasicBlock *PhiBB = Phi.getParent();
+ IRBuilder<> Builder(PhiBB, PhiBB->getFirstInsertionPt());
+
+ Value *Ctlz64 = Builder.CreateIntrinsic(
+ Intrinsic::ctlz, {SrcVal->getType()}, {SrcVal, Builder.getTrue()});
+ Value *Trunc = Builder.CreateTrunc(Ctlz64, Builder.getInt32Ty());
+ Value *IsZero =
+ Builder.CreateICmpEQ(SrcVal, ConstantInt::get(SrcVal->getType(), 0));
+ Value *Result = Builder.CreateSelect(
+ IsZero, ConstantInt::get(Builder.getInt32Ty(), *ZeroResult), Trunc);
+
+ Phi.replaceAllUsesWith(Result);
+ ++NumSplitCTLZFolded;
+ return true;
+ }
+
+ return false;
+}
+
+/// Try to fold a select-based split cttz(32-bit) pattern into a single 64-bit
+/// cttz.
+///
+/// %lo = trunc i64 %val to i32
+/// %cmp = icmp eq i32 %lo, 0
+/// %shr = lshr i64 %val, 32
+/// %hi = trunc i64 %shr to i32
+/// %cttz_hi = call i32 @llvm.cttz.i32(i32 %hi, ...)
+/// %hi_plus32 = add/or i32 %cttz_hi, 32
+/// %cttz_lo = call i32 @llvm.cttz.i32(i32 %lo, ...)
+/// %result = select i1 %cmp, i32 %hi_plus32, i32 %cttz_lo
+/// -->
+/// %cttz64 = call i64 @llvm.cttz.i64(i64 %val, i1 false)
+/// %result = trunc i64 %cttz64 to i32
+/// Alive prrof: https://alive2.llvm.org/ce/z/2MkVVM
+static bool foldSelectSplitCTTZ(Instruction &I) {
+ Value *Cond, *TrueVal, *FalseVal;
+ if (!match(&I, m_Select(m_Value(Cond), m_Value(TrueVal), m_Value(FalseVal))))
+ return false;
+
+ if (!I.getType()->isIntegerTy(32))
+ return false;
+
+ // select (icmp eq (trunc SrcVal to i32), 0), HiResult, LoResult
+ // Or select (icmp ne ...), LoResult, HiResult
+ Value *LoTrunc;
+ Value *HiResult, *LoResult;
+ if (match(Cond,
+ m_SpecificICmp(CmpInst::ICMP_EQ, m_Value(LoTrunc), m_ZeroInt()))) {
+ HiResult = TrueVal;
+ LoResult = FalseVal;
+ } else if (match(Cond, m_SpecificICmp(CmpInst::ICMP_NE, m_Value(LoTrunc),
+ m_ZeroInt()))) {
+ HiResult = FalseVal;
+ LoResult = TrueVal;
+ } else {
+ return false;
+ }
+
+ // LoTrunc: trunc i64 SrcVal to i32
+ Value *SrcVal;
+ if (!match(LoTrunc, m_Trunc(m_Value(SrcVal))))
+ return false;
+ if (!SrcVal->getType()->isIntegerTy(64))
+ return false;
+
+ // LoResult: cttz.i32(trunc(SrcVal), _), must use same truncated value
+ Value *LoCttzArg;
+ if (!match(LoResult,
+ m_Intrinsic<Intrinsic::cttz>(m_Value(LoCttzArg), m_Value())))
+ return false;
+ if (LoCttzArg != LoTrunc)
+ return false;
+
+ // HiResult: add/or(cttz.i32(trunc(lshr(SrcVal, 32)), _), 32)
+ Value *CttzHiCall;
+ if (!match(HiResult, m_c_Add(m_Value(CttzHiCall), m_SpecificInt(32))) &&
+ !match(HiResult, m_c_Or(m_Value(CttzHiCall), m_SpecificInt(32))))
+ return false;
+
+ Value *HiCttzArg;
+ if (!match(CttzHiCall,
+ m_Intrinsic<Intrinsic::cttz>(m_Value(HiCttzArg), m_Value())))
+ return false;
+
+ if (!match(HiCttzArg, m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(32)))))
+ return false;
+
+ // Match successful.
+ IRBuilder<> Builder(&I);
+ Value *Cttz64 = Builder.CreateIntrinsic(Intrinsic::cttz, {SrcVal->getType()},
+ {SrcVal, Builder.getFalse()});
+ Value *Trunc = Builder.CreateTrunc(Cttz64, Builder.getInt32Ty());
+
+ I.replaceAllUsesWith(Trunc);
+ ++NumSelectCTTZFolded;
+ return true;
+}
+
+/// Same as foldSelectSplitCTTZ but for leading zeros (ctlz).
+///
+/// %shr = lshr i64 %val, 32
+/// %hi = trunc i64 %shr to i32
+/// %cmp = icmp eq i32 %hi, 0 (or icmp eq i64 %shr, 0)
+/// %lo = trunc i64 %val to i32
+/// %ctlz_lo = call i32 @llvm.ctlz.i32(i32 %lo, ...)
+/// %lo_plus32 = add/or i32 %ctlz_lo, 32
+/// %ctlz_hi = call i32 @llvm.ctlz.i32(i32 %hi, ...)
+/// %result = select i1 %cmp, i32 %lo_plus32, i32 %ctlz_hi
+/// -->
+/// %ctlz64 = call i64 @llvm.ctlz.i64(i64 %val, i1 false)
+/// %result = trunc i64 %ctlz64 to i32
+///
+/// Alive proof: https://alive2.llvm.org/ce/z/rBJygf
+static bool foldSelectSplitCTLZ(Instruction &I) {
+ Value *Cond, *TrueVal, *FalseVal;
+ if (!match(&I, m_Select(m_Value(Cond), m_Value(TrueVal), m_Value(FalseVal))))
+ return false;
+
+ if (!I.getType()->isIntegerTy(32))
+ return false;
+
+ // select (icmp eq HiPart, 0), LoResult, HiResult
+ // HiPart could be (trunc (lshr SrcVal, 32) to i32) or (lshr SrcVal, 32)
+ Value *HiPart;
+ Value *LoResult, *HiResult;
+ if (match(Cond,
+ m_SpecificICmp(CmpInst::ICMP_EQ, m_Value(HiPart), m_ZeroInt()))) {
+ LoResult = TrueVal; // upper is zero
+ HiResult = FalseVal; // upper non-zero
+ } else if (match(Cond, m_SpecificICmp(CmpInst::ICMP_NE, m_Value(HiPart),
+ m_ZeroInt()))) {
+ LoResult = FalseVal;
+ HiResult = TrueVal;
+ } else {
+ return false;
+ }
+
+ // Extract SrcVal from HiPart: either trunc(lshr(SrcVal, 32)) or
+ // lshr(SrcVal, 32)
+ Value *SrcVal;
+ if (match(HiPart, m_Trunc(m_LShr(m_Value(SrcVal), m_SpecificInt(32))))) {
+ // HiPart is trunc(lshr(SrcVal, 32))
+ } else if (match(HiPart, m_LShr(m_Value(SrcVal), m_SpecificInt(32)))) {
+ // HiPart is lshr(SrcVal, 32)
+ } else {
+ return false;
+ }
+ if (!SrcVal->getType()->isIntegerTy(64))
+ return false;
+
+ // HiResult: ctlz.i32(trunc(lshr(SrcVal,32)), _)
+ Value *HiCtlzArg;
+ if (!match(HiResult,
+ m_Intrinsic<Intrinsic::ctlz>(m_Value(HiCtlzArg), m_Value())))
+ return false;
+
+ // HiCtlzArg should be a trunc of lshr(SrcVal, 32)
+ if (!match(HiCtlzArg, m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(32)))))
+ return false;
+
+ // LoResult: add/or(ctlz.i32(trunc(SrcVal), _), 32)
+ Value *CtlzLoCall;
+ if (!match(LoResult, m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(32))) &&
+ !match(LoResult, m_c_Or(m_Value(CtlzLoCall), m_SpecificInt(32))))
+ return false;
+
+ Value *LoCtlzArg;
+ if (!match(CtlzLoCall,
+ m_Intrinsic<Intrinsic::ctlz>(m_Value(LoCtlzArg), m_Value())))
+ return false;
+
+ if (!match(LoCtlzArg, m_Trunc(m_Specific(SrcVal))))
+ return false;
+
+ // Match successsful.
+ IRBuilder<> Builder(&I);
+ Value *Ctlz64 = Builder.CreateIntrinsic(Intrinsic::ctlz, {SrcVal->getType()},
+ {SrcVal, Builder.getFalse()});
+ Value *Trunc = Builder.CreateTrunc(Ctlz64, Builder.getInt32Ty());
+
+ I.replaceAllUsesWith(Trunc);
+ ++NumSelectCTLZFolded;
+ return true;
+}
+
/// Match a pattern for a bitwise funnel/rotate operation that partially guards
/// against undefined behavior by branching around the funnel-shift/rotation
/// when the shift amount is 0.
@@ -2023,6 +2464,10 @@ static bool foldUnusualPatterns(Function &F, DominatorTree &DT,
for (Instruction &I : make_early_inc_range(llvm::reverse(BB))) {
MadeChange |= foldAnyOrAllBitsSet(I);
MadeChange |= foldGuardedFunnelShift(I, DT);
+ MadeChange |= foldSplitWidthCTTZ(I, DT);
+ MadeChange |= foldSplitWidthCTLZ(I, DT);
+ MadeChange |= foldSelectSplitCTTZ(I);
+ MadeChange |= foldSelectSplitCTLZ(I);
MadeChange |= tryToRecognizePopCount(I);
MadeChange |= tryToFPToSat(I, TTI);
MadeChange |= tryToRecognizeTableBasedCttz(I, DL);
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
index 3015d9ece004c..db5296dcf05ca 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
@@ -10,18 +10,14 @@ define i32 @split_ctlz_phi(i64 noundef %val) {
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: [[CONV1:%.*]] = trunc nuw i64 [[VAL]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV1]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i32 [[TMP0]], 32
-; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 63, i32 [[TMP2]]
; CHECK-NEXT: br label [[CLEANUP:%.*]]
; CHECK: if.else:
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP4:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV]], i1 true)
; CHECK-NEXT: br label [[CLEANUP]]
; CHECK: cleanup:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 63, i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[TMP3]]
;
entry:
@@ -55,18 +51,14 @@ define i32 @split_ctlz_phi_add(i64 noundef %val) {
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: [[CONV1:%.*]] = trunc nuw i64 [[VAL]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV1]], i1 false)
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP0]], 32
-; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 63, i32 [[TMP2]]
; CHECK-NEXT: br label [[CLEANUP:%.*]]
; CHECK: if.else:
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV]], i1 true)
; CHECK-NEXT: br label [[CLEANUP]]
; CHECK: cleanup:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 63, i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[TMP3]]
;
entry:
@@ -96,14 +88,8 @@ cleanup:
define i32 @split_ctlz_select(i64 noundef %val) {
; CHECK-LABEL: @split_ctlz_select(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
-; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
-; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP2]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
@@ -122,14 +108,8 @@ entry:
define i32 @split_ctlz_select_i64cmp(i64 noundef %val) {
; CHECK-LABEL: @split_ctlz_select_i64cmp(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
-; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP2]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
@@ -148,14 +128,8 @@ entry:
define i32 @split_ctlz_select_ne(i64 noundef %val) {
; CHECK-LABEL: @split_ctlz_select_ne(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
-; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[CONV_HI]], 0
-; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[TMP2]], i32 [[ADD]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
@@ -174,14 +148,8 @@ entry:
define i32 @split_ctlz_select_or(i64 noundef %val) {
; CHECK-LABEL: @split_ctlz_select_or(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
-; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
-; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = or disjoint i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP2]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
index 45212cd53dc96..b228bf6576baf 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
@@ -10,18 +10,14 @@ define i32 @split_cttz_phi(i64 noundef %val) {
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: [[SHR:%.*]] = lshr exact i64 [[VAL]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i32 [[TMP0]], 32
-; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 32, i32 [[TMP2]]
; CHECK-NEXT: br label [[CLEANUP:%.*]]
; CHECK: if.else:
-; CHECK-NEXT: [[TMP4:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
; CHECK-NEXT: br label [[CLEANUP]]
; CHECK: cleanup:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 32, i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[TMP3]]
;
entry:
@@ -55,18 +51,14 @@ define i32 @split_cttz_phi_add(i64 noundef %val) {
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: [[SHR:%.*]] = lshr exact i64 [[VAL]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP0]], 32
-; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 32, i32 [[TMP2]]
; CHECK-NEXT: br label [[CLEANUP:%.*]]
; CHECK: if.else:
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
; CHECK-NEXT: br label [[CLEANUP]]
; CHECK: cleanup:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 32, i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[TMP3]]
;
entry:
@@ -97,15 +89,9 @@ cleanup:
define i32 @split_cttz_select(i64 noundef %val) {
; CHECK-LABEL: @split_cttz_select(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP2]], 32
-; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
-; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP3]]
-; CHECK-NEXT: ret i32 [[RETVAL]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
%conv1 = trunc i64 %val to i32
@@ -123,15 +109,9 @@ entry:
define i32 @split_cttz_select_or(i64 noundef %val) {
; CHECK-LABEL: @split_cttz_select_or(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = or disjoint i32 [[TMP2]], 32
-; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
-; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP3]]
-; CHECK-NEXT: ret i32 [[RETVAL]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
%conv1 = trunc i64 %val to i32
@@ -149,15 +129,9 @@ entry:
define i32 @split_cttz_select_ne(i64 noundef %val) {
; CHECK-LABEL: @split_cttz_select_ne(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[TMP1]], 0
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP2]], 32
-; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
-; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[TMP3]], i32 [[ADD]]
-; CHECK-NEXT: ret i32 [[RETVAL]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
%conv1 = trunc i64 %val to i32
@@ -175,15 +149,9 @@ entry:
define i32 @split_cttz_select_shift(i64 noundef %val) {
; CHECK-LABEL: @split_cttz_select_shift(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP2]], 32
-; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
-; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP3]]
-; CHECK-NEXT: [[RESULT:%.*]] = lshr i32 [[RETVAL]], 3
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: [[RESULT:%.*]] = lshr i32 [[TMP1]], 3
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
>From ec035ccebf3ff153108537a4519e4a935e5d1a88 Mon Sep 17 00:00:00 2001
From: Usha Gupta <usha.gupta at arm.com>
Date: Thu, 16 Apr 2026 11:13:54 +0000
Subject: [PATCH 3/3] Address review comments: generalized the fold for integer
types, removed dominance check and modified to match on or disjoint
---
.../AggressiveInstCombine.cpp | 299 +++++++++---------
.../AggressiveInstCombine/fold-split-ctlz.ll | 130 ++++++++
.../AggressiveInstCombine/fold-split-cttz.ll | 131 ++++++++
3 files changed, 418 insertions(+), 142 deletions(-)
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index bb5cacf43ad9f..06dba966cca4c 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -76,43 +76,48 @@ static cl::opt<unsigned>
cl::desc("The maximum length of a constant string to "
"inline a memchr call."));
-/// Try to fold a phi that merges two 32-bit cttz operations into a single
-/// 64-bit cttz. This pattern arises when a 64-bit cttz is implemented as
-/// two 32-bit cttz operations with a branch on whether the low half is zero:
+/// Try to fold a phi that merges two half-width cttz operations into a single
+/// full-width cttz. This pattern arises when a wide cttz is implemented as
+/// two half-width cttz operations with a branch on whether the low half is
+/// zero:
///
/// GuardBB:
-/// %lo = trunc i64 %val to i32
-/// %cmp = icmp eq i32 %lo, 0
+/// %lo = trunc iN %val to i(N/2)
+/// %cmp = icmp eq i(N/2) %lo, 0
/// br i1 %cmp, label %HiBB, label %LoBB
///
/// HiBB:
-/// %shr = lshr i64 %val, 32
-/// %hi = trunc i64 %shr to i32
-/// %cttz_hi = call i32 @llvm.cttz.i32(i32 %hi, i1 false)
-/// %val_is_zero = icmp eq i64 %val, 0
-/// %cttz_hi_plus32 = or i32 %cttz_hi, 32
-/// %hi_result = select i1 %val_is_zero, i32 ZeroResult, i32 %cttz_hi_plus32
+/// %shr = lshr iN %val, N/2
+/// %hi = trunc iN %shr to i(N/2)
+/// %cttz_hi = call i(N/2) @llvm.cttz.i(N/2)(i(N/2) %hi, i1 false)
+/// %val_is_zero = icmp eq iN %val, 0
+/// %cttz_hi_plus = add/or_disjoint i(N/2) %cttz_hi, N/2
+/// %hi_result = select i1 %val_is_zero, i(N/2) ZeroResult, i(N/2)
+/// %cttz_hi_plus
/// br label %PhiBB
///
/// LoBB:
-/// %cttz_lo = call i32 @llvm.cttz.i32(i32 %lo, i1 true)
+/// %cttz_lo = call i(N/2) @llvm.cttz.i(N/2)(i(N/2) %lo, i1 true)
/// br label %PhiBB
///
/// PhiBB:
-/// %result = phi i32 [%hi_result, %HiBB], [%cttz_lo, %LoBB]
+/// %result = phi i(N/2) [%hi_result, %HiBB], [%cttz_lo, %LoBB]
/// -->
-/// %cttz64 = call i64 @llvm.cttz.i64(i64 %val, i1 true)
-/// %trunc = trunc i64 %cttz64 to i32
-/// %val_is_zero = icmp eq i64 %val, 0
-/// %result = select i1 %val_is_zero, i32 ZeroResult, i32 %trunc
+/// %cttz_wide = call iN @llvm.cttz.iN(iN %val, i1 true)
+/// %trunc = trunc iN %cttz_wide to i(N/2)
+/// %val_is_zero = icmp eq iN %val, 0
+/// %result = select i1 %val_is_zero, i(N/2) ZeroResult, i(N/2) %trunc
///
-/// Alive proof: https://alive2.llvm.org/ce/z/FHgJpq
-static bool foldSplitWidthCTTZ(Instruction &I, const DominatorTree &DT) {
+/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/FHgJpq
+static bool foldSplitWidthCTTZ(Instruction &I) {
if (I.getOpcode() != Instruction::PHI || I.getNumOperands() != 2)
return false;
- if (!I.getType()->isIntegerTy(32))
+ Type *HalfTy = I.getType();
+ if (!HalfTy->isIntegerTy())
return false;
+ unsigned HalfWidth = HalfTy->getIntegerBitWidth();
+ unsigned FullWidth = HalfWidth * 2;
PHINode &Phi = cast<PHINode>(I);
@@ -124,22 +129,21 @@ static bool foldSplitWidthCTTZ(Instruction &I, const DominatorTree &DT) {
BasicBlock *HiBB = Phi.getIncomingBlock(HiIdx);
BasicBlock *LoBB = Phi.getIncomingBlock(LoIdx);
- // Match LoVal: cttz.i32(trunc(SrcVal to i32), _)
+ // Match LoVal: cttz.i(N/2)(trunc(SrcVal to i(N/2)), _)
Value *LoTrunc;
if (!match(LoVal,
m_Intrinsic<Intrinsic::cttz>(m_Value(LoTrunc), m_Value())))
continue;
- // LoTrunc must be a trunc of an i64 value.
+ // LoTrunc must be a trunc of a value with double the width.
Value *SrcVal = nullptr;
if (!match(LoTrunc, m_Trunc(m_Value(SrcVal))))
continue;
- if (!SrcVal->getType()->isIntegerTy(64))
+ if (!SrcVal->getType()->isIntegerTy(FullWidth))
continue;
- // Match HiVal: select(icmp eq SrcVal, 0, C,
- // or(cttz.i32(trunc(lshr(SrcVal, 32)), _), 32))
- // where C is the result when the full value is zero.
+ // Match HiVal: select(icmp eq SrcVal, 0, ZeroResult,
+ // add/or_disjoint(cttz(trunc(lshr(SrcVal, N/2))), N/2))
Value *SelectCond, *SelectTrue, *SelectFalse;
if (!match(HiVal, m_Select(m_Value(SelectCond), m_Value(SelectTrue),
m_Value(SelectFalse))))
@@ -155,52 +159,49 @@ static bool foldSplitWidthCTTZ(Instruction &I, const DominatorTree &DT) {
if (!match(SelectTrue, m_APInt(ZeroResult)))
continue;
- // SelectFalse: or(cttz.i32(trunc(lshr(SrcVal, 32)), _), 32)
- // Also accept add instead of or.
+ // SelectFalse: add/or_disjoint(cttz(trunc(lshr(SrcVal, N/2))), N/2)
Value *CttzHiCall;
- if (!match(SelectFalse, m_c_Or(m_Value(CttzHiCall), m_SpecificInt(32))) &&
- !match(SelectFalse, m_c_Add(m_Value(CttzHiCall), m_SpecificInt(32))))
+ if (!match(SelectFalse,
+ m_c_Add(m_Value(CttzHiCall), m_SpecificInt(HalfWidth))) &&
+ !match(SelectFalse,
+ m_c_DisjointOr(m_Value(CttzHiCall), m_SpecificInt(HalfWidth))))
continue;
- // CttzHiCall: cttz.i32(trunc(lshr(SrcVal, 32)), _)
+ // CttzHiCall: cttz(trunc(lshr(SrcVal, N/2)), _)
Value *HiTrunc;
if (!match(CttzHiCall,
m_Intrinsic<Intrinsic::cttz>(m_Value(HiTrunc), m_Value())))
continue;
- if (!match(HiTrunc, m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(32)))))
+ if (!match(HiTrunc,
+ m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(HalfWidth)))))
continue;
// Both HiBB and LoBB share a single predecessor
- // that branches on whether the low 32 bits are zero.
+ // that branches on whether the low half is zero.
BasicBlock *GuardBB = LoBB->getSinglePredecessor();
if (!GuardBB || GuardBB != HiBB->getSinglePredecessor())
continue;
Instruction *GuardTerm = GuardBB->getTerminator();
- // Guard: br (icmp eq (trunc SrcVal to i32), 0), HiBB, LoBB
+ // Guard: br (icmp eq (trunc SrcVal to i(N/2)), 0), HiBB, LoBB
if (!match(GuardTerm,
m_Br(m_SpecificICmp(CmpInst::ICMP_EQ,
m_Trunc(m_Specific(SrcVal)), m_ZeroInt()),
m_SpecificBB(HiBB), m_SpecificBB(LoBB))))
continue;
- // Verify SrcVal dominates the guard.
- if (auto *SrcI = dyn_cast<Instruction>(SrcVal))
- if (!DT.dominates(SrcI, GuardTerm))
- continue;
-
// Match successful.
BasicBlock *PhiBB = Phi.getParent();
IRBuilder<> Builder(PhiBB, PhiBB->getFirstInsertionPt());
- Value *Cttz64 = Builder.CreateIntrinsic(
+ Value *CttzWide = Builder.CreateIntrinsic(
Intrinsic::cttz, {SrcVal->getType()}, {SrcVal, Builder.getTrue()});
- Value *Trunc = Builder.CreateTrunc(Cttz64, Builder.getInt32Ty());
+ Value *Trunc = Builder.CreateTrunc(CttzWide, HalfTy);
Value *IsZero =
Builder.CreateICmpEQ(SrcVal, ConstantInt::get(SrcVal->getType(), 0));
Value *Result = Builder.CreateSelect(
- IsZero, ConstantInt::get(Builder.getInt32Ty(), *ZeroResult), Trunc);
+ IsZero, ConstantInt::get(HalfTy, *ZeroResult), Trunc);
Phi.replaceAllUsesWith(Result);
++NumSplitCTTZFolded;
@@ -210,42 +211,47 @@ static bool foldSplitWidthCTTZ(Instruction &I, const DominatorTree &DT) {
return false;
}
-/// Try to fold a phi that merges two 32-bit ctlz operations into a single
-/// 64-bit ctlz. This is similar to foldSplitWidthCTTZ, but for leading zeros.
+/// Try to fold a phi that merges two half-width ctlz operations into a single
+/// full-width ctlz. This is similar to foldSplitWidthCTTZ, but for leading
+/// zeros.
///
/// GuardBB:
-/// %shr = lshr i64 %val, 32
-/// %cmp = icmp eq i64 %shr, 0
+/// %shr = lshr iN %val, N/2
+/// %cmp = icmp eq iN %shr, 0
/// br i1 %cmp, label %LoBB, label %HiBB
///
-/// LoBB (upper is zero: count in lower half + 32):
-/// %lo = trunc i64 %val to i32
-/// %ctlz_lo = call i32 @llvm.ctlz.i32(i32 %lo, i1 true)
-/// %val_is_zero = icmp eq i64 %val, 0
-/// %ctlz_lo_plus32 = or i32 %ctlz_lo, 32
-/// %lo_result = select i1 %val_is_zero, i32 ZeroResult, i32 %ctlz_lo_plus32
+/// LoBB (upper is zero: count in lower half + N/2):
+/// %lo = trunc iN %val to i(N/2)
+/// %ctlz_lo = call i(N/2) @llvm.ctlz.i(N/2)(i(N/2) %lo, i1 true)
+/// %val_is_zero = icmp eq iN %val, 0
+/// %ctlz_lo_plus = add/or_disjoint i(N/2) %ctlz_lo, N/2
+/// %lo_result = select i1 %val_is_zero, i(N/2) ZeroResult, i(N/2)
+/// %ctlz_lo_plus
/// br label %PhiBB
///
/// HiBB (upper is non-zero: count in upper half):
-/// %hi = trunc i64 %shr to i32
-/// %ctlz_hi = call i32 @llvm.ctlz.i32(i32 %hi, i1 true)
+/// %hi = trunc iN %shr to i(N/2)
+/// %ctlz_hi = call i(N/2) @llvm.ctlz.i(N/2)(i(N/2) %hi, i1 true)
/// br label %PhiBB
///
/// PhiBB:
-/// %result = phi i32 [%lo_result, %LoBB], [%ctlz_hi, %HiBB]
+/// %result = phi i(N/2) [%lo_result, %LoBB], [%ctlz_hi, %HiBB]
/// -->
-/// %ctlz64 = call i64 @llvm.ctlz.i64(i64 %val, i1 true)
-/// %trunc = trunc i64 %ctlz64 to i32
-/// %val_is_zero = icmp eq i64 %val, 0
-/// %result = select i1 %val_is_zero, i32 ZeroResult, i32 %trunc
+/// %ctlz_wide = call iN @llvm.ctlz.iN(iN %val, i1 true)
+/// %trunc = trunc iN %ctlz_wide to i(N/2)
+/// %val_is_zero = icmp eq iN %val, 0
+/// %result = select i1 %val_is_zero, i(N/2) ZeroResult, i(N/2) %trunc
///
-/// Alive proof: https://alive2.llvm.org/ce/z/peel2Y
-static bool foldSplitWidthCTLZ(Instruction &I, const DominatorTree &DT) {
+/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/peel2Y
+static bool foldSplitWidthCTLZ(Instruction &I) {
if (I.getOpcode() != Instruction::PHI || I.getNumOperands() != 2)
return false;
- if (!I.getType()->isIntegerTy(32))
+ Type *HalfTy = I.getType();
+ if (!HalfTy->isIntegerTy())
return false;
+ unsigned HalfWidth = HalfTy->getIntegerBitWidth();
+ unsigned FullWidth = HalfWidth * 2;
PHINode &Phi = cast<PHINode>(I);
@@ -257,22 +263,23 @@ static bool foldSplitWidthCTLZ(Instruction &I, const DominatorTree &DT) {
BasicBlock *LoBB = Phi.getIncomingBlock(LoIdx);
BasicBlock *HiBB = Phi.getIncomingBlock(HiIdx);
- // Match HiVal: ctlz.i32(trunc(lshr(SrcVal, 32) to i32), _)
- // This is the ctlz on the upper 32 bits (when upper is non-zero).
+ // Match HiVal: ctlz(trunc(lshr(SrcVal, N/2)), _)
+ // This is the ctlz on the upper half (when upper is non-zero).
Value *HiTrunc;
if (!match(HiVal,
m_Intrinsic<Intrinsic::ctlz>(m_Value(HiTrunc), m_Value())))
continue;
- // HiTrunc must be trunc(lshr(SrcVal, 32)).
+ // HiTrunc must be trunc(lshr(SrcVal, N/2)).
Value *SrcVal = nullptr;
- if (!match(HiTrunc, m_Trunc(m_LShr(m_Value(SrcVal), m_SpecificInt(32)))))
+ if (!match(HiTrunc,
+ m_Trunc(m_LShr(m_Value(SrcVal), m_SpecificInt(HalfWidth)))))
continue;
- if (!SrcVal->getType()->isIntegerTy(64))
+ if (!SrcVal->getType()->isIntegerTy(FullWidth))
continue;
// Match LoVal: select(icmp eq SrcVal, 0, ZeroResult,
- // or/add(ctlz.i32(trunc(SrcVal to i32), _), 32))
+ // add/or_disjoint(ctlz(trunc(SrcVal), _), N/2))
Value *SelectCond, *SelectTrue, *SelectFalse;
if (!match(LoVal, m_Select(m_Value(SelectCond), m_Value(SelectTrue),
m_Value(SelectFalse))))
@@ -288,13 +295,15 @@ static bool foldSplitWidthCTLZ(Instruction &I, const DominatorTree &DT) {
if (!match(SelectTrue, m_APInt(ZeroResult)))
continue;
- // SelectFalse: or/add(ctlz.i32(trunc(SrcVal to i32), _), 32)
+ // SelectFalse: add/or_disjoint(ctlz(trunc(SrcVal), _), N/2)
Value *CtlzLoCall;
- if (!match(SelectFalse, m_c_Or(m_Value(CtlzLoCall), m_SpecificInt(32))) &&
- !match(SelectFalse, m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(32))))
+ if (!match(SelectFalse,
+ m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(HalfWidth))) &&
+ !match(SelectFalse,
+ m_c_DisjointOr(m_Value(CtlzLoCall), m_SpecificInt(HalfWidth))))
continue;
- // CtlzLoCall: ctlz.i32(trunc(SrcVal to i32), _)
+ // CtlzLoCall: ctlz(trunc(SrcVal), _)
Value *LoTrunc;
if (!match(CtlzLoCall,
m_Intrinsic<Intrinsic::ctlz>(m_Value(LoTrunc), m_Value())))
@@ -304,36 +313,31 @@ static bool foldSplitWidthCTLZ(Instruction &I, const DominatorTree &DT) {
continue;
// Both LoBB and HiBB share a single predecessor that branches on whether
- // the upper 32 bits are zero.
+ // the upper half is zero.
BasicBlock *GuardBB = LoBB->getSinglePredecessor();
if (!GuardBB || GuardBB != HiBB->getSinglePredecessor())
continue;
Instruction *GuardTerm = GuardBB->getTerminator();
- // Guard: br (icmp eq (lshr SrcVal, 32), 0), LoBB, HiBB
+ // Guard: br (icmp eq (lshr SrcVal, N/2), 0), LoBB, HiBB
if (!match(GuardTerm, m_Br(m_SpecificICmp(CmpInst::ICMP_EQ,
m_LShr(m_Specific(SrcVal),
- m_SpecificInt(32)),
+ m_SpecificInt(HalfWidth)),
m_ZeroInt()),
m_SpecificBB(LoBB), m_SpecificBB(HiBB))))
continue;
- // Verify SrcVal dominates the guard.
- if (auto *SrcI = dyn_cast<Instruction>(SrcVal))
- if (!DT.dominates(SrcI, GuardTerm))
- continue;
-
// Match successful.
BasicBlock *PhiBB = Phi.getParent();
IRBuilder<> Builder(PhiBB, PhiBB->getFirstInsertionPt());
- Value *Ctlz64 = Builder.CreateIntrinsic(
+ Value *CtlzWide = Builder.CreateIntrinsic(
Intrinsic::ctlz, {SrcVal->getType()}, {SrcVal, Builder.getTrue()});
- Value *Trunc = Builder.CreateTrunc(Ctlz64, Builder.getInt32Ty());
+ Value *Trunc = Builder.CreateTrunc(CtlzWide, HalfTy);
Value *IsZero =
Builder.CreateICmpEQ(SrcVal, ConstantInt::get(SrcVal->getType(), 0));
Value *Result = Builder.CreateSelect(
- IsZero, ConstantInt::get(Builder.getInt32Ty(), *ZeroResult), Trunc);
+ IsZero, ConstantInt::get(HalfTy, *ZeroResult), Trunc);
Phi.replaceAllUsesWith(Result);
++NumSplitCTLZFolded;
@@ -343,30 +347,32 @@ static bool foldSplitWidthCTLZ(Instruction &I, const DominatorTree &DT) {
return false;
}
-/// Try to fold a select-based split cttz(32-bit) pattern into a single 64-bit
-/// cttz.
+/// Try to fold a select-based split cttz pattern into a single full-width cttz.
///
-/// %lo = trunc i64 %val to i32
-/// %cmp = icmp eq i32 %lo, 0
-/// %shr = lshr i64 %val, 32
-/// %hi = trunc i64 %shr to i32
-/// %cttz_hi = call i32 @llvm.cttz.i32(i32 %hi, ...)
-/// %hi_plus32 = add/or i32 %cttz_hi, 32
-/// %cttz_lo = call i32 @llvm.cttz.i32(i32 %lo, ...)
-/// %result = select i1 %cmp, i32 %hi_plus32, i32 %cttz_lo
+/// %lo = trunc iN %val to i(N/2)
+/// %cmp = icmp eq i(N/2) %lo, 0
+/// %shr = lshr iN %val, N/2
+/// %hi = trunc iN %shr to i(N/2)
+/// %cttz_hi = call i(N/2) @llvm.cttz.i(N/2)(i(N/2) %hi, ...)
+/// %hi_plus = add/or_disjoint i(N/2) %cttz_hi, N/2
+/// %cttz_lo = call i(N/2) @llvm.cttz.i(N/2)(i(N/2) %lo, ...)
+/// %result = select i1 %cmp, i(N/2) %hi_plus, i(N/2) %cttz_lo
/// -->
-/// %cttz64 = call i64 @llvm.cttz.i64(i64 %val, i1 false)
-/// %result = trunc i64 %cttz64 to i32
-/// Alive prrof: https://alive2.llvm.org/ce/z/2MkVVM
+/// %cttz_wide = call iN @llvm.cttz.iN(iN %val, i1 false)
+/// %result = trunc iN %cttz_wide to i(N/2)
+/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/2MkVVM
static bool foldSelectSplitCTTZ(Instruction &I) {
Value *Cond, *TrueVal, *FalseVal;
if (!match(&I, m_Select(m_Value(Cond), m_Value(TrueVal), m_Value(FalseVal))))
return false;
- if (!I.getType()->isIntegerTy(32))
+ Type *HalfTy = I.getType();
+ if (!HalfTy->isIntegerTy())
return false;
+ unsigned HalfWidth = HalfTy->getIntegerBitWidth();
+ unsigned FullWidth = HalfWidth * 2;
- // select (icmp eq (trunc SrcVal to i32), 0), HiResult, LoResult
+ // select (icmp eq (trunc SrcVal to i(N/2)), 0), HiResult, LoResult
// Or select (icmp ne ...), LoResult, HiResult
Value *LoTrunc;
Value *HiResult, *LoResult;
@@ -382,14 +388,14 @@ static bool foldSelectSplitCTTZ(Instruction &I) {
return false;
}
- // LoTrunc: trunc i64 SrcVal to i32
+ // LoTrunc: trunc iN SrcVal to i(N/2)
Value *SrcVal;
if (!match(LoTrunc, m_Trunc(m_Value(SrcVal))))
return false;
- if (!SrcVal->getType()->isIntegerTy(64))
+ if (!SrcVal->getType()->isIntegerTy(FullWidth))
return false;
- // LoResult: cttz.i32(trunc(SrcVal), _), must use same truncated value
+ // LoResult: cttz(trunc(SrcVal), _), must use same truncated value
Value *LoCttzArg;
if (!match(LoResult,
m_Intrinsic<Intrinsic::cttz>(m_Value(LoCttzArg), m_Value())))
@@ -397,10 +403,12 @@ static bool foldSelectSplitCTTZ(Instruction &I) {
if (LoCttzArg != LoTrunc)
return false;
- // HiResult: add/or(cttz.i32(trunc(lshr(SrcVal, 32)), _), 32)
+ // HiResult: add/or_disjoint(cttz(trunc(lshr(SrcVal, N/2)), _), N/2)
Value *CttzHiCall;
- if (!match(HiResult, m_c_Add(m_Value(CttzHiCall), m_SpecificInt(32))) &&
- !match(HiResult, m_c_Or(m_Value(CttzHiCall), m_SpecificInt(32))))
+ if (!match(HiResult,
+ m_c_Add(m_Value(CttzHiCall), m_SpecificInt(HalfWidth))) &&
+ !match(HiResult,
+ m_c_DisjointOr(m_Value(CttzHiCall), m_SpecificInt(HalfWidth))))
return false;
Value *HiCttzArg;
@@ -408,14 +416,15 @@ static bool foldSelectSplitCTTZ(Instruction &I) {
m_Intrinsic<Intrinsic::cttz>(m_Value(HiCttzArg), m_Value())))
return false;
- if (!match(HiCttzArg, m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(32)))))
+ if (!match(HiCttzArg,
+ m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(HalfWidth)))))
return false;
// Match successful.
IRBuilder<> Builder(&I);
- Value *Cttz64 = Builder.CreateIntrinsic(Intrinsic::cttz, {SrcVal->getType()},
- {SrcVal, Builder.getFalse()});
- Value *Trunc = Builder.CreateTrunc(Cttz64, Builder.getInt32Ty());
+ Value *CttzWide = Builder.CreateIntrinsic(
+ Intrinsic::cttz, {SrcVal->getType()}, {SrcVal, Builder.getFalse()});
+ Value *Trunc = Builder.CreateTrunc(CttzWide, HalfTy);
I.replaceAllUsesWith(Trunc);
++NumSelectCTTZFolded;
@@ -424,35 +433,38 @@ static bool foldSelectSplitCTTZ(Instruction &I) {
/// Same as foldSelectSplitCTTZ but for leading zeros (ctlz).
///
-/// %shr = lshr i64 %val, 32
-/// %hi = trunc i64 %shr to i32
-/// %cmp = icmp eq i32 %hi, 0 (or icmp eq i64 %shr, 0)
-/// %lo = trunc i64 %val to i32
-/// %ctlz_lo = call i32 @llvm.ctlz.i32(i32 %lo, ...)
-/// %lo_plus32 = add/or i32 %ctlz_lo, 32
-/// %ctlz_hi = call i32 @llvm.ctlz.i32(i32 %hi, ...)
-/// %result = select i1 %cmp, i32 %lo_plus32, i32 %ctlz_hi
+/// %shr = lshr iN %val, N/2
+/// %hi = trunc iN %shr to i(N/2)
+/// %cmp = icmp eq i(N/2) %hi, 0 (or icmp eq iN %shr, 0)
+/// %lo = trunc iN %val to i(N/2)
+/// %ctlz_lo = call i(N/2) @llvm.ctlz.i(N/2)(i(N/2) %lo, ...)
+/// %lo_plus = add/or_disjoint i(N/2) %ctlz_lo, N/2
+/// %ctlz_hi = call i(N/2) @llvm.ctlz.i(N/2)(i(N/2) %hi, ...)
+/// %result = select i1 %cmp, i(N/2) %lo_plus, i(N/2) %ctlz_hi
/// -->
-/// %ctlz64 = call i64 @llvm.ctlz.i64(i64 %val, i1 false)
-/// %result = trunc i64 %ctlz64 to i32
+/// %ctlz_wide = call iN @llvm.ctlz.iN(iN %val, i1 false)
+/// %result = trunc iN %ctlz_wide to i(N/2)
///
-/// Alive proof: https://alive2.llvm.org/ce/z/rBJygf
+/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/rBJygf
static bool foldSelectSplitCTLZ(Instruction &I) {
Value *Cond, *TrueVal, *FalseVal;
if (!match(&I, m_Select(m_Value(Cond), m_Value(TrueVal), m_Value(FalseVal))))
return false;
- if (!I.getType()->isIntegerTy(32))
+ Type *HalfTy = I.getType();
+ if (!HalfTy->isIntegerTy())
return false;
+ unsigned HalfWidth = HalfTy->getIntegerBitWidth();
+ unsigned FullWidth = HalfWidth * 2;
// select (icmp eq HiPart, 0), LoResult, HiResult
- // HiPart could be (trunc (lshr SrcVal, 32) to i32) or (lshr SrcVal, 32)
+ // HiPart could be (trunc (lshr SrcVal, N/2) to i(N/2)) or (lshr SrcVal, N/2)
Value *HiPart;
Value *LoResult, *HiResult;
if (match(Cond,
m_SpecificICmp(CmpInst::ICMP_EQ, m_Value(HiPart), m_ZeroInt()))) {
- LoResult = TrueVal; // upper is zero
- HiResult = FalseVal; // upper non-zero
+ LoResult = TrueVal; // upper is zero: count in lower + N/2
+ HiResult = FalseVal; // upper non-zero: count in upper
} else if (match(Cond, m_SpecificICmp(CmpInst::ICMP_NE, m_Value(HiPart),
m_ZeroInt()))) {
LoResult = FalseVal;
@@ -461,33 +473,36 @@ static bool foldSelectSplitCTLZ(Instruction &I) {
return false;
}
- // Extract SrcVal from HiPart: either trunc(lshr(SrcVal, 32)) or
- // lshr(SrcVal, 32)
+ // Extract SrcVal from HiPart: either trunc(lshr(SrcVal, N/2)) or
+ // lshr(SrcVal, N/2)
Value *SrcVal;
- if (match(HiPart, m_Trunc(m_LShr(m_Value(SrcVal), m_SpecificInt(32))))) {
- // HiPart is trunc(lshr(SrcVal, 32))
- } else if (match(HiPart, m_LShr(m_Value(SrcVal), m_SpecificInt(32)))) {
- // HiPart is lshr(SrcVal, 32)
+ if (match(HiPart,
+ m_Trunc(m_LShr(m_Value(SrcVal), m_SpecificInt(HalfWidth))))) {
+ // HiPart is trunc(lshr(SrcVal, N/2))
+ } else if (match(HiPart, m_LShr(m_Value(SrcVal), m_SpecificInt(HalfWidth)))) {
+ // HiPart is lshr(SrcVal, N/2)
} else {
return false;
}
- if (!SrcVal->getType()->isIntegerTy(64))
+ if (!SrcVal->getType()->isIntegerTy(FullWidth))
return false;
- // HiResult: ctlz.i32(trunc(lshr(SrcVal,32)), _)
+ // HiResult: ctlz(trunc(lshr(SrcVal, N/2)), _)
Value *HiCtlzArg;
if (!match(HiResult,
m_Intrinsic<Intrinsic::ctlz>(m_Value(HiCtlzArg), m_Value())))
return false;
- // HiCtlzArg should be a trunc of lshr(SrcVal, 32)
- if (!match(HiCtlzArg, m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(32)))))
+ if (!match(HiCtlzArg,
+ m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(HalfWidth)))))
return false;
- // LoResult: add/or(ctlz.i32(trunc(SrcVal), _), 32)
+ // LoResult: add/or_disjoint(ctlz(trunc(SrcVal), _), N/2)
Value *CtlzLoCall;
- if (!match(LoResult, m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(32))) &&
- !match(LoResult, m_c_Or(m_Value(CtlzLoCall), m_SpecificInt(32))))
+ if (!match(LoResult,
+ m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(HalfWidth))) &&
+ !match(LoResult,
+ m_c_DisjointOr(m_Value(CtlzLoCall), m_SpecificInt(HalfWidth))))
return false;
Value *LoCtlzArg;
@@ -498,11 +513,11 @@ static bool foldSelectSplitCTLZ(Instruction &I) {
if (!match(LoCtlzArg, m_Trunc(m_Specific(SrcVal))))
return false;
- // Match successsful.
+ // Match successful.
IRBuilder<> Builder(&I);
- Value *Ctlz64 = Builder.CreateIntrinsic(Intrinsic::ctlz, {SrcVal->getType()},
- {SrcVal, Builder.getFalse()});
- Value *Trunc = Builder.CreateTrunc(Ctlz64, Builder.getInt32Ty());
+ Value *CtlzWide = Builder.CreateIntrinsic(
+ Intrinsic::ctlz, {SrcVal->getType()}, {SrcVal, Builder.getFalse()});
+ Value *Trunc = Builder.CreateTrunc(CtlzWide, HalfTy);
I.replaceAllUsesWith(Trunc);
++NumSelectCTLZFolded;
@@ -2464,8 +2479,8 @@ static bool foldUnusualPatterns(Function &F, DominatorTree &DT,
for (Instruction &I : make_early_inc_range(llvm::reverse(BB))) {
MadeChange |= foldAnyOrAllBitsSet(I);
MadeChange |= foldGuardedFunnelShift(I, DT);
- MadeChange |= foldSplitWidthCTTZ(I, DT);
- MadeChange |= foldSplitWidthCTLZ(I, DT);
+ MadeChange |= foldSplitWidthCTTZ(I);
+ MadeChange |= foldSplitWidthCTLZ(I);
MadeChange |= foldSelectSplitCTTZ(I);
MadeChange |= foldSelectSplitCTLZ(I);
MadeChange |= tryToRecognizePopCount(I);
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
index db5296dcf05ca..df6ef312d27c1 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
@@ -164,6 +164,107 @@ entry:
ret i32 %retval
}
+; i32 split into two i16 ctlz (phi-based).
+define i16 @split_ctlz_phi_i32(i32 noundef %val) {
+; CHECK-LABEL: @split_ctlz_phi_i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[VAL:%.*]], 16
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[SHR]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: br label [[CLEANUP:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: br label [[CLEANUP]]
+; CHECK: cleanup:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[VAL]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i16
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[VAL]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i16 16, i16 [[TMP1]]
+; CHECK-NEXT: ret i16 [[TMP3]]
+;
+entry:
+ %shr = lshr i32 %val, 16
+ %cmp = icmp eq i32 %shr, 0
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+ %conv_lo = trunc i32 %val to i16
+ %0 = call i16 @llvm.ctlz.i16(i16 %conv_lo, i1 true)
+ %1 = icmp eq i32 %val, 0
+ %2 = add i16 %0, 16
+ %add = select i1 %1, i16 16, i16 %2
+ br label %cleanup
+
+if.else:
+ %conv_hi = trunc nuw i32 %shr to i16
+ %3 = call i16 @llvm.ctlz.i16(i16 %conv_hi, i1 true)
+ br label %cleanup
+
+cleanup:
+ %retval.0 = phi i16 [ %add, %if.then ], [ %3, %if.else ]
+ ret i16 %retval.0
+}
+
+; i32 split into two i16 ctlz (select-based).
+define i16 @split_ctlz_select_i32(i32 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i16
+; CHECK-NEXT: ret i16 [[TMP1]]
+;
+entry:
+ %shr = lshr i32 %val, 16
+ %conv_hi = trunc nuw i32 %shr to i16
+ %cmp = icmp eq i16 %conv_hi, 0
+ %conv_lo = trunc i32 %val to i16
+ %0 = call i16 @llvm.ctlz.i16(i16 %conv_lo, i1 false)
+ %add = add nuw nsw i16 %0, 16
+ %1 = call i16 @llvm.ctlz.i16(i16 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i16 %add, i16 %1
+ ret i16 %retval
+}
+
+; i128 split into two i64 ctlz (select-based).
+define i64 @split_ctlz_select_i128(i128 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_i128(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i128 @llvm.ctlz.i128(i128 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i128 [[TMP0]] to i64
+; CHECK-NEXT: ret i64 [[TMP1]]
+;
+entry:
+ %shr = lshr i128 %val, 64
+ %conv_hi = trunc nuw i128 %shr to i64
+ %cmp = icmp eq i64 %conv_hi, 0
+ %conv_lo = trunc i128 %val to i64
+ %0 = call i64 @llvm.ctlz.i64(i64 %conv_lo, i1 false)
+ %add = add nuw nsw i64 %0, 64
+ %1 = call i64 @llvm.ctlz.i64(i64 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i64 %add, i64 %1
+ ret i64 %retval
+}
+
+; i16 split into two i8 ctlz (select-based).
+define i8 @split_ctlz_select_i16(i16 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_i16(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i16 @llvm.ctlz.i16(i16 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i16 [[TMP0]] to i8
+; CHECK-NEXT: ret i8 [[TMP1]]
+;
+entry:
+ %shr = lshr i16 %val, 8
+ %conv_hi = trunc nuw i16 %shr to i8
+ %cmp = icmp eq i8 %conv_hi, 0
+ %conv_lo = trunc i16 %val to i8
+ %0 = call i8 @llvm.ctlz.i8(i8 %conv_lo, i1 false)
+ %add = add nuw nsw i8 %0, 8
+ %1 = call i8 @llvm.ctlz.i8(i8 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i8 %add, i8 %1
+ ret i8 %retval
+}
+
; Negative test: mismatched source values, should NOT be folded.
define i32 @split_ctlz_different_sources(i64 noundef %val, i64 noundef %val2) {
; CHECK-LABEL: @split_ctlz_different_sources(
@@ -216,5 +317,34 @@ entry:
ret i32 %retval
}
+; Negative test: plain or (without disjoint flag), should NOT be folded
+; in select-based pattern because or(32, 32)=32 != add(32, 32)=64.
+define i32 @split_ctlz_select_plain_or(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_plain_or(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %conv_hi = trunc nuw i64 %shr to i32
+ %cmp = icmp eq i32 %conv_hi, 0
+ %conv_lo = trunc i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = or i32 %0, 32
+ %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+declare i8 @llvm.ctlz.i8(i8, i1 immarg)
+declare i16 @llvm.ctlz.i16(i16, i1 immarg)
declare i32 @llvm.ctlz.i32(i32, i1 immarg)
declare i64 @llvm.ctlz.i64(i64, i1 immarg)
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
index b228bf6576baf..15879ec05ab8f 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
@@ -167,6 +167,108 @@ entry:
ret i32 %result
}
+; i32 split into two i16 cttz (phi-based).
+define i16 @split_cttz_phi_i32(i32 noundef %val) {
+; CHECK-LABEL: @split_cttz_phi_i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i32 [[VAL:%.*]] to i16
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i16 [[CONV1]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: br label [[CLEANUP:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: br label [[CLEANUP]]
+; CHECK: cleanup:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[VAL]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i16
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[VAL]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i16 16, i16 [[TMP1]]
+; CHECK-NEXT: ret i16 [[TMP3]]
+;
+entry:
+ %conv1 = trunc i32 %val to i16
+ %cmp = icmp eq i16 %conv1, 0
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+ %shr = lshr i32 %val, 16
+ %conv = trunc nuw i32 %shr to i16
+ %0 = call i16 @llvm.cttz.i16(i16 %conv, i1 true)
+ %1 = icmp eq i32 %val, 0
+ %2 = add i16 %0, 16
+ %add = select i1 %1, i16 16, i16 %2
+ br label %cleanup
+
+if.else:
+ %3 = call i16 @llvm.cttz.i16(i16 %conv1, i1 true)
+ br label %cleanup
+
+cleanup:
+ %retval.0 = phi i16 [ %add, %if.then ], [ %3, %if.else ]
+ ret i16 %retval.0
+}
+
+; i32 split into two i16 cttz (select-based).
+define i16 @split_cttz_select_i32(i32 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i16
+; CHECK-NEXT: ret i16 [[TMP1]]
+;
+entry:
+ %conv1 = trunc i32 %val to i16
+ %cmp = icmp eq i16 %conv1, 0
+ %shr = lshr i32 %val, 16
+ %conv = trunc nuw i32 %shr to i16
+ %0 = call i16 @llvm.cttz.i16(i16 %conv, i1 false)
+ %add = add nuw nsw i16 %0, 16
+ %1 = call i16 @llvm.cttz.i16(i16 %conv1, i1 true)
+ %retval = select i1 %cmp, i16 %add, i16 %1
+ ret i16 %retval
+}
+
+; i128 split into two i64 cttz (select-based).
+define i64 @split_cttz_select_i128(i128 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_i128(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i128 @llvm.cttz.i128(i128 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i128 [[TMP0]] to i64
+; CHECK-NEXT: ret i64 [[TMP1]]
+;
+entry:
+ %conv1 = trunc i128 %val to i64
+ %cmp = icmp eq i64 %conv1, 0
+ %shr = lshr i128 %val, 64
+ %conv = trunc nuw i128 %shr to i64
+ %0 = call i64 @llvm.cttz.i64(i64 %conv, i1 false)
+ %add = add nuw nsw i64 %0, 64
+ %1 = call i64 @llvm.cttz.i64(i64 %conv1, i1 true)
+ %retval = select i1 %cmp, i64 %add, i64 %1
+ ret i64 %retval
+}
+
+; i16 split into two i8 cttz (select-based).
+define i8 @split_cttz_select_i16(i16 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_i16(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i16 @llvm.cttz.i16(i16 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i16 [[TMP0]] to i8
+; CHECK-NEXT: ret i8 [[TMP1]]
+;
+entry:
+ %conv1 = trunc i16 %val to i8
+ %cmp = icmp eq i8 %conv1, 0
+ %shr = lshr i16 %val, 8
+ %conv = trunc nuw i16 %shr to i8
+ %0 = call i8 @llvm.cttz.i8(i8 %conv, i1 false)
+ %add = add nuw nsw i8 %0, 8
+ %1 = call i8 @llvm.cttz.i8(i8 %conv1, i1 true)
+ %retval = select i1 %cmp, i8 %add, i8 %1
+ ret i8 %retval
+}
+
+
; Negative test: mismatched source values, should NOT be folded.
define i32 @split_cttz_different_sources(i64 noundef %val, i64 noundef %val2) {
; CHECK-LABEL: @split_cttz_different_sources(
@@ -219,5 +321,34 @@ entry:
ret i32 %retval
}
+; Negative test: plain or (without disjoint flag), should NOT be folded
+; in select-based pattern because or(32, 32)=32 != add(32, 32)=64.
+define i32 @split_cttz_select_plain_or(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_plain_or(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ %shr = lshr i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = or i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+declare i8 @llvm.cttz.i8(i8, i1 immarg)
+declare i16 @llvm.cttz.i16(i16, i1 immarg)
declare i32 @llvm.cttz.i32(i32, i1 immarg)
declare i64 @llvm.cttz.i64(i64, i1 immarg)
More information about the llvm-commits
mailing list