[llvm] [AggressiveInstCombine] Fold split-width i32 cttz/ctlz patterns into wide i64 intrinsics (PR #192296)
Usha Gupta via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 23 04:09:41 PDT 2026
https://github.com/usha1830 updated https://github.com/llvm/llvm-project/pull/192296
>From 29a700e28572dcd70d1a8a931f707479689b66bd Mon Sep 17 00:00:00 2001
From: Usha Gupta <usha.gupta at arm.com>
Date: Wed, 15 Apr 2026 16:22:22 +0000
Subject: [PATCH 1/6] Pre-commit tests for folding split-width i32 cttz/ctlz
into wide (i64) intrinsics
---
.../AggressiveInstCombine/fold-split-ctlz.ll | 252 +++++++++++++++++
.../AggressiveInstCombine/fold-split-cttz.ll | 255 ++++++++++++++++++
2 files changed, 507 insertions(+)
create mode 100644 llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
create mode 100644 llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
new file mode 100644
index 0000000000000..3015d9ece004c
--- /dev/null
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
@@ -0,0 +1,252 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=aggressive-instcombine -S | FileCheck %s
+
+; Phi-based split i64 ctlz: branch on whether upper 32 bits are zero,
+; ctlz each half separately, merge via phi.
+define i32 @split_ctlz_phi(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_phi(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc nuw i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i32 [[TMP0]], 32
+; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 63, i32 [[TMP2]]
+; CHECK-NEXT: br label [[CLEANUP:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV]], i1 true)
+; CHECK-NEXT: br label [[CLEANUP]]
+; CHECK: cleanup:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %cmp = icmp eq i64 %shr, 0
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+ %conv1 = trunc nuw i64 %val to i32
+ %0 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv1, i1 true)
+ %1 = icmp eq i64 %val, 0
+ %2 = or disjoint i32 %0, 32
+ %add = select i1 %1, i32 63, i32 %2
+ br label %cleanup
+
+if.else:
+ %conv = trunc nuw i64 %shr to i32
+ %3 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv, i1 true)
+ br label %cleanup
+
+cleanup:
+ %retval.0 = phi i32 [ %add, %if.then ], [ %3, %if.else ]
+ ret i32 %retval.0
+}
+
+; Same pattern but with add instead of or.
+define i32 @split_ctlz_phi_add(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_phi_add(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc nuw i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV1]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP0]], 32
+; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 63, i32 [[TMP2]]
+; CHECK-NEXT: br label [[CLEANUP:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV]], i1 true)
+; CHECK-NEXT: br label [[CLEANUP]]
+; CHECK: cleanup:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %cmp = icmp eq i64 %shr, 0
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+ %conv1 = trunc nuw i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv1, i1 false)
+ %1 = icmp eq i64 %val, 0
+ %2 = add i32 %0, 32
+ %add = select i1 %1, i32 63, i32 %2
+ br label %cleanup
+
+if.else:
+ %conv = trunc nuw i64 %shr to i32
+ %3 = call i32 @llvm.ctlz.i32(i32 %conv, i1 true)
+ br label %cleanup
+
+cleanup:
+ %retval.0 = phi i32 [ %add, %if.then ], [ %3, %if.else ]
+ ret i32 %retval.0
+}
+
+; Select-based split i64 ctlz
+define i32 @split_ctlz_select(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP2]]
+; CHECK-NEXT: ret i32 [[TMP1]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %conv_hi = trunc nuw i64 %shr to i32
+ %cmp = icmp eq i32 %conv_hi, 0
+ %conv_lo = trunc nuw i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Select-based with i64 comparison on lshr result (not truncated).
+define i32 @split_ctlz_select_i64cmp(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_i64cmp(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP2]]
+; CHECK-NEXT: ret i32 [[TMP1]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %cmp = icmp eq i64 %shr, 0
+ %conv_hi = trunc nuw i64 %shr to i32
+ %conv_lo = trunc nuw i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Select-based with icmp ne (inverted condition).
+define i32 @split_ctlz_select_ne(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_ne(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[CONV_HI]], 0
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[TMP2]], i32 [[ADD]]
+; CHECK-NEXT: ret i32 [[TMP1]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %conv_hi = trunc nuw i64 %shr to i32
+ %cmp = icmp ne i32 %conv_hi, 0
+ %conv_lo = trunc nuw i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %1, i32 %add
+ ret i32 %retval
+}
+
+; Select-based with or instead of add.
+define i32 @split_ctlz_select_or(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_or(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = or disjoint i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP2]]
+; CHECK-NEXT: ret i32 [[TMP1]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %conv_hi = trunc nuw i64 %shr to i32
+ %cmp = icmp eq i32 %conv_hi, 0
+ %conv_lo = trunc nuw i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = or disjoint i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Negative test: mismatched source values, should NOT be folded.
+define i32 @split_ctlz_different_sources(i64 noundef %val, i64 noundef %val2) {
+; CHECK-LABEL: @split_ctlz_different_sources(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL2:%.*]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %conv_hi = trunc nuw i64 %shr to i32
+ %cmp = icmp eq i32 %conv_hi, 0
+ %conv_lo = trunc nuw i64 %val2 to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Negative test: wrong shift amount, should NOT be folded.
+define i32 @split_ctlz_wrong_shift(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_wrong_shift(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 16
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc i64 [[SHR]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %shr = lshr i64 %val, 16
+ %conv_hi = trunc i64 %shr to i32
+ %cmp = icmp eq i32 %conv_hi, 0
+ %conv_lo = trunc i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+declare i32 @llvm.ctlz.i32(i32, i1 immarg)
+declare i64 @llvm.ctlz.i64(i64, i1 immarg)
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
new file mode 100644
index 0000000000000..45212cd53dc96
--- /dev/null
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
@@ -0,0 +1,255 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=aggressive-instcombine -S | FileCheck %s
+
+; Phi-based split i64 cttz: branch on whether low 32 bits are zero,
+; cttz each half separately, merge via phi.
+define i32 @split_cttz_phi(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_phi(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[SHR:%.*]] = lshr exact i64 [[VAL]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i32 [[TMP0]], 32
+; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 32, i32 [[TMP2]]
+; CHECK-NEXT: br label [[CLEANUP:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP4:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: br label [[CLEANUP]]
+; CHECK: cleanup:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+ %shr = lshr exact i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv, i1 true)
+ %1 = icmp eq i64 %val, 0
+ %2 = or disjoint i32 %0, 32
+ %add = select i1 %1, i32 32, i32 %2
+ br label %cleanup
+
+if.else:
+ %3 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ br label %cleanup
+
+cleanup:
+ %retval.0 = phi i32 [ %add, %if.then ], [ %3, %if.else ]
+ ret i32 %retval.0
+}
+
+; Same pattern but with add instead of or for the +32.
+define i32 @split_cttz_phi_add(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_phi_add(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[SHR:%.*]] = lshr exact i64 [[VAL]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP0]], 32
+; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 32, i32 [[TMP2]]
+; CHECK-NEXT: br label [[CLEANUP:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: br label [[CLEANUP]]
+; CHECK: cleanup:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+ %shr = lshr exact i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %1 = icmp eq i64 %val, 0
+ %2 = add i32 %0, 32
+ %add = select i1 %1, i32 32, i32 %2
+ br label %cleanup
+
+if.else:
+ %3 = call i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ br label %cleanup
+
+cleanup:
+ %retval.0 = phi i32 [ %add, %if.then ], [ %3, %if.else ]
+ ret i32 %retval.0
+}
+
+; Select-based split i64 cttz: both cttz.i32 are speculated,
+; result picked by select.
+define i32 @split_cttz_select(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_select(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP2]], 32
+; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP3]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ %shr = lshr i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Select-based with or instead of add.
+define i32 @split_cttz_select_or(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_or(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = or disjoint i32 [[TMP2]], 32
+; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP3]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ %shr = lshr i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = or disjoint i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Select-based with icmp ne (inverted condition).
+define i32 @split_cttz_select_ne(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_ne(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP2]], 32
+; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[TMP3]], i32 [[ADD]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp ne i32 %conv1, 0
+ %shr = lshr i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %1, i32 %add
+ ret i32 %retval
+}
+
+; Select-based cttz used inside a larger expression (>> 3)
+define i32 @split_cttz_select_shift(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_shift(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP2]], 32
+; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP3]]
+; CHECK-NEXT: [[RESULT:%.*]] = lshr i32 [[RETVAL]], 3
+; CHECK-NEXT: ret i32 [[RESULT]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ %shr = lshr i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ %result = lshr i32 %retval, 3
+ ret i32 %result
+}
+
+; Negative test: mismatched source values, should NOT be folded.
+define i32 @split_cttz_different_sources(i64 noundef %val, i64 noundef %val2) {
+; CHECK-LABEL: @split_cttz_different_sources(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL2:%.*]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ %shr = lshr i64 %val2, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+; Negative test: shift amount is not 32, should NOT be folded.
+define i32 @split_cttz_wrong_shift(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_wrong_shift(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL]], 16
+; CHECK-NEXT: [[CONV:%.*]] = trunc i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ %shr = lshr i64 %val, 16
+ %conv = trunc i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+declare i32 @llvm.cttz.i32(i32, i1 immarg)
+declare i64 @llvm.cttz.i64(i64, i1 immarg)
>From a1cd7e88caa9abb74875b5256e0645ade75931a8 Mon Sep 17 00:00:00 2001
From: Usha Gupta <usha.gupta at arm.com>
Date: Wed, 15 Apr 2026 16:42:11 +0000
Subject: [PATCH 2/6] Fold split-width (i32) cttz/ctlz patterns into wide (i64)
intrinsics
---
.../AggressiveInstCombine.cpp | 445 ++++++++++++++++++
.../AggressiveInstCombine/fold-split-ctlz.ll | 64 +--
.../AggressiveInstCombine/fold-split-cttz.ll | 72 +--
3 files changed, 481 insertions(+), 100 deletions(-)
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index 7e57c5eff0bf1..bb5cacf43ad9f 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -53,6 +53,14 @@ STATISTIC(NumGuardedRotates,
STATISTIC(NumGuardedFunnelShifts,
"Number of guarded funnel shifts transformed into funnel shifts");
STATISTIC(NumPopCountRecognized, "Number of popcount idioms recognized");
+STATISTIC(NumSplitCTTZFolded,
+ "Number of split-width cttz patterns folded into full-width cttz");
+STATISTIC(NumSplitCTLZFolded,
+ "Number of split-width ctlz patterns folded into full-width ctlz");
+STATISTIC(NumSelectCTTZFolded,
+ "Number of select-based split cttz patterns folded");
+STATISTIC(NumSelectCTLZFolded,
+ "Number of select-based split ctlz patterns folded");
static cl::opt<unsigned> MaxInstrsToScan(
"aggressive-instcombine-max-scan-instrs", cl::init(64), cl::Hidden,
@@ -68,6 +76,439 @@ static cl::opt<unsigned>
cl::desc("The maximum length of a constant string to "
"inline a memchr call."));
+/// Try to fold a phi that merges two 32-bit cttz operations into a single
+/// 64-bit cttz. This pattern arises when a 64-bit cttz is implemented as
+/// two 32-bit cttz operations with a branch on whether the low half is zero:
+///
+/// GuardBB:
+/// %lo = trunc i64 %val to i32
+/// %cmp = icmp eq i32 %lo, 0
+/// br i1 %cmp, label %HiBB, label %LoBB
+///
+/// HiBB:
+/// %shr = lshr i64 %val, 32
+/// %hi = trunc i64 %shr to i32
+/// %cttz_hi = call i32 @llvm.cttz.i32(i32 %hi, i1 false)
+/// %val_is_zero = icmp eq i64 %val, 0
+/// %cttz_hi_plus32 = or i32 %cttz_hi, 32
+/// %hi_result = select i1 %val_is_zero, i32 ZeroResult, i32 %cttz_hi_plus32
+/// br label %PhiBB
+///
+/// LoBB:
+/// %cttz_lo = call i32 @llvm.cttz.i32(i32 %lo, i1 true)
+/// br label %PhiBB
+///
+/// PhiBB:
+/// %result = phi i32 [%hi_result, %HiBB], [%cttz_lo, %LoBB]
+/// -->
+/// %cttz64 = call i64 @llvm.cttz.i64(i64 %val, i1 true)
+/// %trunc = trunc i64 %cttz64 to i32
+/// %val_is_zero = icmp eq i64 %val, 0
+/// %result = select i1 %val_is_zero, i32 ZeroResult, i32 %trunc
+///
+/// Alive proof: https://alive2.llvm.org/ce/z/FHgJpq
+static bool foldSplitWidthCTTZ(Instruction &I, const DominatorTree &DT) {
+ if (I.getOpcode() != Instruction::PHI || I.getNumOperands() != 2)
+ return false;
+
+ if (!I.getType()->isIntegerTy(32))
+ return false;
+
+ PHINode &Phi = cast<PHINode>(I);
+
+ // Try both orderings of phi operands.
+ for (unsigned HiIdx = 0; HiIdx < 2; ++HiIdx) {
+ unsigned LoIdx = 1 - HiIdx;
+ Value *HiVal = Phi.getIncomingValue(HiIdx);
+ Value *LoVal = Phi.getIncomingValue(LoIdx);
+ BasicBlock *HiBB = Phi.getIncomingBlock(HiIdx);
+ BasicBlock *LoBB = Phi.getIncomingBlock(LoIdx);
+
+ // Match LoVal: cttz.i32(trunc(SrcVal to i32), _)
+ Value *LoTrunc;
+ if (!match(LoVal,
+ m_Intrinsic<Intrinsic::cttz>(m_Value(LoTrunc), m_Value())))
+ continue;
+
+ // LoTrunc must be a trunc of an i64 value.
+ Value *SrcVal = nullptr;
+ if (!match(LoTrunc, m_Trunc(m_Value(SrcVal))))
+ continue;
+ if (!SrcVal->getType()->isIntegerTy(64))
+ continue;
+
+ // Match HiVal: select(icmp eq SrcVal, 0, C,
+ // or(cttz.i32(trunc(lshr(SrcVal, 32)), _), 32))
+ // where C is the result when the full value is zero.
+ Value *SelectCond, *SelectTrue, *SelectFalse;
+ if (!match(HiVal, m_Select(m_Value(SelectCond), m_Value(SelectTrue),
+ m_Value(SelectFalse))))
+ continue;
+
+ // SelectCond: icmp eq SrcVal, 0
+ if (!match(SelectCond, m_SpecificICmp(CmpInst::ICMP_EQ, m_Specific(SrcVal),
+ m_ZeroInt())))
+ continue;
+
+ // SelectTrue: the zero-input result
+ const APInt *ZeroResult;
+ if (!match(SelectTrue, m_APInt(ZeroResult)))
+ continue;
+
+ // SelectFalse: or(cttz.i32(trunc(lshr(SrcVal, 32)), _), 32)
+ // Also accept add instead of or.
+ Value *CttzHiCall;
+ if (!match(SelectFalse, m_c_Or(m_Value(CttzHiCall), m_SpecificInt(32))) &&
+ !match(SelectFalse, m_c_Add(m_Value(CttzHiCall), m_SpecificInt(32))))
+ continue;
+
+ // CttzHiCall: cttz.i32(trunc(lshr(SrcVal, 32)), _)
+ Value *HiTrunc;
+ if (!match(CttzHiCall,
+ m_Intrinsic<Intrinsic::cttz>(m_Value(HiTrunc), m_Value())))
+ continue;
+
+ if (!match(HiTrunc, m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(32)))))
+ continue;
+
+ // Both HiBB and LoBB share a single predecessor
+ // that branches on whether the low 32 bits are zero.
+ BasicBlock *GuardBB = LoBB->getSinglePredecessor();
+ if (!GuardBB || GuardBB != HiBB->getSinglePredecessor())
+ continue;
+
+ Instruction *GuardTerm = GuardBB->getTerminator();
+ // Guard: br (icmp eq (trunc SrcVal to i32), 0), HiBB, LoBB
+ if (!match(GuardTerm,
+ m_Br(m_SpecificICmp(CmpInst::ICMP_EQ,
+ m_Trunc(m_Specific(SrcVal)), m_ZeroInt()),
+ m_SpecificBB(HiBB), m_SpecificBB(LoBB))))
+ continue;
+
+ // Verify SrcVal dominates the guard.
+ if (auto *SrcI = dyn_cast<Instruction>(SrcVal))
+ if (!DT.dominates(SrcI, GuardTerm))
+ continue;
+
+ // Match successful.
+ BasicBlock *PhiBB = Phi.getParent();
+ IRBuilder<> Builder(PhiBB, PhiBB->getFirstInsertionPt());
+
+ Value *Cttz64 = Builder.CreateIntrinsic(
+ Intrinsic::cttz, {SrcVal->getType()}, {SrcVal, Builder.getTrue()});
+ Value *Trunc = Builder.CreateTrunc(Cttz64, Builder.getInt32Ty());
+ Value *IsZero =
+ Builder.CreateICmpEQ(SrcVal, ConstantInt::get(SrcVal->getType(), 0));
+ Value *Result = Builder.CreateSelect(
+ IsZero, ConstantInt::get(Builder.getInt32Ty(), *ZeroResult), Trunc);
+
+ Phi.replaceAllUsesWith(Result);
+ ++NumSplitCTTZFolded;
+ return true;
+ }
+
+ return false;
+}
+
+/// Try to fold a phi that merges two 32-bit ctlz operations into a single
+/// 64-bit ctlz. This is similar to foldSplitWidthCTTZ, but for leading zeros.
+///
+/// GuardBB:
+/// %shr = lshr i64 %val, 32
+/// %cmp = icmp eq i64 %shr, 0
+/// br i1 %cmp, label %LoBB, label %HiBB
+///
+/// LoBB (upper is zero: count in lower half + 32):
+/// %lo = trunc i64 %val to i32
+/// %ctlz_lo = call i32 @llvm.ctlz.i32(i32 %lo, i1 true)
+/// %val_is_zero = icmp eq i64 %val, 0
+/// %ctlz_lo_plus32 = or i32 %ctlz_lo, 32
+/// %lo_result = select i1 %val_is_zero, i32 ZeroResult, i32 %ctlz_lo_plus32
+/// br label %PhiBB
+///
+/// HiBB (upper is non-zero: count in upper half):
+/// %hi = trunc i64 %shr to i32
+/// %ctlz_hi = call i32 @llvm.ctlz.i32(i32 %hi, i1 true)
+/// br label %PhiBB
+///
+/// PhiBB:
+/// %result = phi i32 [%lo_result, %LoBB], [%ctlz_hi, %HiBB]
+/// -->
+/// %ctlz64 = call i64 @llvm.ctlz.i64(i64 %val, i1 true)
+/// %trunc = trunc i64 %ctlz64 to i32
+/// %val_is_zero = icmp eq i64 %val, 0
+/// %result = select i1 %val_is_zero, i32 ZeroResult, i32 %trunc
+///
+/// Alive proof: https://alive2.llvm.org/ce/z/peel2Y
+static bool foldSplitWidthCTLZ(Instruction &I, const DominatorTree &DT) {
+ if (I.getOpcode() != Instruction::PHI || I.getNumOperands() != 2)
+ return false;
+
+ if (!I.getType()->isIntegerTy(32))
+ return false;
+
+ PHINode &Phi = cast<PHINode>(I);
+
+ // Try both orderings of phi operands.
+ for (unsigned HiIdx = 0; HiIdx < 2; ++HiIdx) {
+ unsigned LoIdx = 1 - HiIdx;
+ Value *LoVal = Phi.getIncomingValue(LoIdx);
+ Value *HiVal = Phi.getIncomingValue(HiIdx);
+ BasicBlock *LoBB = Phi.getIncomingBlock(LoIdx);
+ BasicBlock *HiBB = Phi.getIncomingBlock(HiIdx);
+
+ // Match HiVal: ctlz.i32(trunc(lshr(SrcVal, 32) to i32), _)
+ // This is the ctlz on the upper 32 bits (when upper is non-zero).
+ Value *HiTrunc;
+ if (!match(HiVal,
+ m_Intrinsic<Intrinsic::ctlz>(m_Value(HiTrunc), m_Value())))
+ continue;
+
+ // HiTrunc must be trunc(lshr(SrcVal, 32)).
+ Value *SrcVal = nullptr;
+ if (!match(HiTrunc, m_Trunc(m_LShr(m_Value(SrcVal), m_SpecificInt(32)))))
+ continue;
+ if (!SrcVal->getType()->isIntegerTy(64))
+ continue;
+
+ // Match LoVal: select(icmp eq SrcVal, 0, ZeroResult,
+ // or/add(ctlz.i32(trunc(SrcVal to i32), _), 32))
+ Value *SelectCond, *SelectTrue, *SelectFalse;
+ if (!match(LoVal, m_Select(m_Value(SelectCond), m_Value(SelectTrue),
+ m_Value(SelectFalse))))
+ continue;
+
+ // SelectCond: icmp eq SrcVal, 0
+ if (!match(SelectCond, m_SpecificICmp(CmpInst::ICMP_EQ, m_Specific(SrcVal),
+ m_ZeroInt())))
+ continue;
+
+ // SelectTrue: the zero-input result.
+ const APInt *ZeroResult;
+ if (!match(SelectTrue, m_APInt(ZeroResult)))
+ continue;
+
+ // SelectFalse: or/add(ctlz.i32(trunc(SrcVal to i32), _), 32)
+ Value *CtlzLoCall;
+ if (!match(SelectFalse, m_c_Or(m_Value(CtlzLoCall), m_SpecificInt(32))) &&
+ !match(SelectFalse, m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(32))))
+ continue;
+
+ // CtlzLoCall: ctlz.i32(trunc(SrcVal to i32), _)
+ Value *LoTrunc;
+ if (!match(CtlzLoCall,
+ m_Intrinsic<Intrinsic::ctlz>(m_Value(LoTrunc), m_Value())))
+ continue;
+
+ if (!match(LoTrunc, m_Trunc(m_Specific(SrcVal))))
+ continue;
+
+ // Both LoBB and HiBB share a single predecessor that branches on whether
+ // the upper 32 bits are zero.
+ BasicBlock *GuardBB = LoBB->getSinglePredecessor();
+ if (!GuardBB || GuardBB != HiBB->getSinglePredecessor())
+ continue;
+
+ Instruction *GuardTerm = GuardBB->getTerminator();
+ // Guard: br (icmp eq (lshr SrcVal, 32), 0), LoBB, HiBB
+ if (!match(GuardTerm, m_Br(m_SpecificICmp(CmpInst::ICMP_EQ,
+ m_LShr(m_Specific(SrcVal),
+ m_SpecificInt(32)),
+ m_ZeroInt()),
+ m_SpecificBB(LoBB), m_SpecificBB(HiBB))))
+ continue;
+
+ // Verify SrcVal dominates the guard.
+ if (auto *SrcI = dyn_cast<Instruction>(SrcVal))
+ if (!DT.dominates(SrcI, GuardTerm))
+ continue;
+
+ // Match successful.
+ BasicBlock *PhiBB = Phi.getParent();
+ IRBuilder<> Builder(PhiBB, PhiBB->getFirstInsertionPt());
+
+ Value *Ctlz64 = Builder.CreateIntrinsic(
+ Intrinsic::ctlz, {SrcVal->getType()}, {SrcVal, Builder.getTrue()});
+ Value *Trunc = Builder.CreateTrunc(Ctlz64, Builder.getInt32Ty());
+ Value *IsZero =
+ Builder.CreateICmpEQ(SrcVal, ConstantInt::get(SrcVal->getType(), 0));
+ Value *Result = Builder.CreateSelect(
+ IsZero, ConstantInt::get(Builder.getInt32Ty(), *ZeroResult), Trunc);
+
+ Phi.replaceAllUsesWith(Result);
+ ++NumSplitCTLZFolded;
+ return true;
+ }
+
+ return false;
+}
+
+/// Try to fold a select-based split cttz(32-bit) pattern into a single 64-bit
+/// cttz.
+///
+/// %lo = trunc i64 %val to i32
+/// %cmp = icmp eq i32 %lo, 0
+/// %shr = lshr i64 %val, 32
+/// %hi = trunc i64 %shr to i32
+/// %cttz_hi = call i32 @llvm.cttz.i32(i32 %hi, ...)
+/// %hi_plus32 = add/or i32 %cttz_hi, 32
+/// %cttz_lo = call i32 @llvm.cttz.i32(i32 %lo, ...)
+/// %result = select i1 %cmp, i32 %hi_plus32, i32 %cttz_lo
+/// -->
+/// %cttz64 = call i64 @llvm.cttz.i64(i64 %val, i1 false)
+/// %result = trunc i64 %cttz64 to i32
+/// Alive prrof: https://alive2.llvm.org/ce/z/2MkVVM
+static bool foldSelectSplitCTTZ(Instruction &I) {
+ Value *Cond, *TrueVal, *FalseVal;
+ if (!match(&I, m_Select(m_Value(Cond), m_Value(TrueVal), m_Value(FalseVal))))
+ return false;
+
+ if (!I.getType()->isIntegerTy(32))
+ return false;
+
+ // select (icmp eq (trunc SrcVal to i32), 0), HiResult, LoResult
+ // Or select (icmp ne ...), LoResult, HiResult
+ Value *LoTrunc;
+ Value *HiResult, *LoResult;
+ if (match(Cond,
+ m_SpecificICmp(CmpInst::ICMP_EQ, m_Value(LoTrunc), m_ZeroInt()))) {
+ HiResult = TrueVal;
+ LoResult = FalseVal;
+ } else if (match(Cond, m_SpecificICmp(CmpInst::ICMP_NE, m_Value(LoTrunc),
+ m_ZeroInt()))) {
+ HiResult = FalseVal;
+ LoResult = TrueVal;
+ } else {
+ return false;
+ }
+
+ // LoTrunc: trunc i64 SrcVal to i32
+ Value *SrcVal;
+ if (!match(LoTrunc, m_Trunc(m_Value(SrcVal))))
+ return false;
+ if (!SrcVal->getType()->isIntegerTy(64))
+ return false;
+
+ // LoResult: cttz.i32(trunc(SrcVal), _), must use same truncated value
+ Value *LoCttzArg;
+ if (!match(LoResult,
+ m_Intrinsic<Intrinsic::cttz>(m_Value(LoCttzArg), m_Value())))
+ return false;
+ if (LoCttzArg != LoTrunc)
+ return false;
+
+ // HiResult: add/or(cttz.i32(trunc(lshr(SrcVal, 32)), _), 32)
+ Value *CttzHiCall;
+ if (!match(HiResult, m_c_Add(m_Value(CttzHiCall), m_SpecificInt(32))) &&
+ !match(HiResult, m_c_Or(m_Value(CttzHiCall), m_SpecificInt(32))))
+ return false;
+
+ Value *HiCttzArg;
+ if (!match(CttzHiCall,
+ m_Intrinsic<Intrinsic::cttz>(m_Value(HiCttzArg), m_Value())))
+ return false;
+
+ if (!match(HiCttzArg, m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(32)))))
+ return false;
+
+ // Match successful.
+ IRBuilder<> Builder(&I);
+ Value *Cttz64 = Builder.CreateIntrinsic(Intrinsic::cttz, {SrcVal->getType()},
+ {SrcVal, Builder.getFalse()});
+ Value *Trunc = Builder.CreateTrunc(Cttz64, Builder.getInt32Ty());
+
+ I.replaceAllUsesWith(Trunc);
+ ++NumSelectCTTZFolded;
+ return true;
+}
+
+/// Same as foldSelectSplitCTTZ but for leading zeros (ctlz).
+///
+/// %shr = lshr i64 %val, 32
+/// %hi = trunc i64 %shr to i32
+/// %cmp = icmp eq i32 %hi, 0 (or icmp eq i64 %shr, 0)
+/// %lo = trunc i64 %val to i32
+/// %ctlz_lo = call i32 @llvm.ctlz.i32(i32 %lo, ...)
+/// %lo_plus32 = add/or i32 %ctlz_lo, 32
+/// %ctlz_hi = call i32 @llvm.ctlz.i32(i32 %hi, ...)
+/// %result = select i1 %cmp, i32 %lo_plus32, i32 %ctlz_hi
+/// -->
+/// %ctlz64 = call i64 @llvm.ctlz.i64(i64 %val, i1 false)
+/// %result = trunc i64 %ctlz64 to i32
+///
+/// Alive proof: https://alive2.llvm.org/ce/z/rBJygf
+static bool foldSelectSplitCTLZ(Instruction &I) {
+ Value *Cond, *TrueVal, *FalseVal;
+ if (!match(&I, m_Select(m_Value(Cond), m_Value(TrueVal), m_Value(FalseVal))))
+ return false;
+
+ if (!I.getType()->isIntegerTy(32))
+ return false;
+
+ // select (icmp eq HiPart, 0), LoResult, HiResult
+ // HiPart could be (trunc (lshr SrcVal, 32) to i32) or (lshr SrcVal, 32)
+ Value *HiPart;
+ Value *LoResult, *HiResult;
+ if (match(Cond,
+ m_SpecificICmp(CmpInst::ICMP_EQ, m_Value(HiPart), m_ZeroInt()))) {
+ LoResult = TrueVal; // upper is zero
+ HiResult = FalseVal; // upper non-zero
+ } else if (match(Cond, m_SpecificICmp(CmpInst::ICMP_NE, m_Value(HiPart),
+ m_ZeroInt()))) {
+ LoResult = FalseVal;
+ HiResult = TrueVal;
+ } else {
+ return false;
+ }
+
+ // Extract SrcVal from HiPart: either trunc(lshr(SrcVal, 32)) or
+ // lshr(SrcVal, 32)
+ Value *SrcVal;
+ if (match(HiPart, m_Trunc(m_LShr(m_Value(SrcVal), m_SpecificInt(32))))) {
+ // HiPart is trunc(lshr(SrcVal, 32))
+ } else if (match(HiPart, m_LShr(m_Value(SrcVal), m_SpecificInt(32)))) {
+ // HiPart is lshr(SrcVal, 32)
+ } else {
+ return false;
+ }
+ if (!SrcVal->getType()->isIntegerTy(64))
+ return false;
+
+ // HiResult: ctlz.i32(trunc(lshr(SrcVal,32)), _)
+ Value *HiCtlzArg;
+ if (!match(HiResult,
+ m_Intrinsic<Intrinsic::ctlz>(m_Value(HiCtlzArg), m_Value())))
+ return false;
+
+ // HiCtlzArg should be a trunc of lshr(SrcVal, 32)
+ if (!match(HiCtlzArg, m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(32)))))
+ return false;
+
+ // LoResult: add/or(ctlz.i32(trunc(SrcVal), _), 32)
+ Value *CtlzLoCall;
+ if (!match(LoResult, m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(32))) &&
+ !match(LoResult, m_c_Or(m_Value(CtlzLoCall), m_SpecificInt(32))))
+ return false;
+
+ Value *LoCtlzArg;
+ if (!match(CtlzLoCall,
+ m_Intrinsic<Intrinsic::ctlz>(m_Value(LoCtlzArg), m_Value())))
+ return false;
+
+ if (!match(LoCtlzArg, m_Trunc(m_Specific(SrcVal))))
+ return false;
+
+ // Match successsful.
+ IRBuilder<> Builder(&I);
+ Value *Ctlz64 = Builder.CreateIntrinsic(Intrinsic::ctlz, {SrcVal->getType()},
+ {SrcVal, Builder.getFalse()});
+ Value *Trunc = Builder.CreateTrunc(Ctlz64, Builder.getInt32Ty());
+
+ I.replaceAllUsesWith(Trunc);
+ ++NumSelectCTLZFolded;
+ return true;
+}
+
/// Match a pattern for a bitwise funnel/rotate operation that partially guards
/// against undefined behavior by branching around the funnel-shift/rotation
/// when the shift amount is 0.
@@ -2023,6 +2464,10 @@ static bool foldUnusualPatterns(Function &F, DominatorTree &DT,
for (Instruction &I : make_early_inc_range(llvm::reverse(BB))) {
MadeChange |= foldAnyOrAllBitsSet(I);
MadeChange |= foldGuardedFunnelShift(I, DT);
+ MadeChange |= foldSplitWidthCTTZ(I, DT);
+ MadeChange |= foldSplitWidthCTLZ(I, DT);
+ MadeChange |= foldSelectSplitCTTZ(I);
+ MadeChange |= foldSelectSplitCTLZ(I);
MadeChange |= tryToRecognizePopCount(I);
MadeChange |= tryToFPToSat(I, TTI);
MadeChange |= tryToRecognizeTableBasedCttz(I, DL);
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
index 3015d9ece004c..db5296dcf05ca 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
@@ -10,18 +10,14 @@ define i32 @split_ctlz_phi(i64 noundef %val) {
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: [[CONV1:%.*]] = trunc nuw i64 [[VAL]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV1]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i32 [[TMP0]], 32
-; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 63, i32 [[TMP2]]
; CHECK-NEXT: br label [[CLEANUP:%.*]]
; CHECK: if.else:
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP4:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV]], i1 true)
; CHECK-NEXT: br label [[CLEANUP]]
; CHECK: cleanup:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 63, i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[TMP3]]
;
entry:
@@ -55,18 +51,14 @@ define i32 @split_ctlz_phi_add(i64 noundef %val) {
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: [[CONV1:%.*]] = trunc nuw i64 [[VAL]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV1]], i1 false)
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP0]], 32
-; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 63, i32 [[TMP2]]
; CHECK-NEXT: br label [[CLEANUP:%.*]]
; CHECK: if.else:
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV]], i1 true)
; CHECK-NEXT: br label [[CLEANUP]]
; CHECK: cleanup:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 63, i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[TMP3]]
;
entry:
@@ -96,14 +88,8 @@ cleanup:
define i32 @split_ctlz_select(i64 noundef %val) {
; CHECK-LABEL: @split_ctlz_select(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
-; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
-; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP2]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
@@ -122,14 +108,8 @@ entry:
define i32 @split_ctlz_select_i64cmp(i64 noundef %val) {
; CHECK-LABEL: @split_ctlz_select_i64cmp(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
-; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP2]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
@@ -148,14 +128,8 @@ entry:
define i32 @split_ctlz_select_ne(i64 noundef %val) {
; CHECK-LABEL: @split_ctlz_select_ne(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
-; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[CONV_HI]], 0
-; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[TMP2]], i32 [[ADD]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
@@ -174,14 +148,8 @@ entry:
define i32 @split_ctlz_select_or(i64 noundef %val) {
; CHECK-LABEL: @split_ctlz_select_or(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
-; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
-; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = or disjoint i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP2:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP2]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
index 45212cd53dc96..b228bf6576baf 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
@@ -10,18 +10,14 @@ define i32 @split_cttz_phi(i64 noundef %val) {
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: [[SHR:%.*]] = lshr exact i64 [[VAL]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i32 [[TMP0]], 32
-; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 32, i32 [[TMP2]]
; CHECK-NEXT: br label [[CLEANUP:%.*]]
; CHECK: if.else:
-; CHECK-NEXT: [[TMP4:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
; CHECK-NEXT: br label [[CLEANUP]]
; CHECK: cleanup:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 32, i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[TMP3]]
;
entry:
@@ -55,18 +51,14 @@ define i32 @split_cttz_phi_add(i64 noundef %val) {
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: [[SHR:%.*]] = lshr exact i64 [[VAL]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[VAL]], 0
-; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP0]], 32
-; CHECK-NEXT: [[ADD:%.*]] = select i1 [[TMP1]], i32 32, i32 [[TMP2]]
; CHECK-NEXT: br label [[CLEANUP:%.*]]
; CHECK: if.else:
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
; CHECK-NEXT: br label [[CLEANUP]]
; CHECK: cleanup:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[TMP4]], [[IF_ELSE]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[VAL]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 32, i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[TMP3]]
;
entry:
@@ -97,15 +89,9 @@ cleanup:
define i32 @split_cttz_select(i64 noundef %val) {
; CHECK-LABEL: @split_cttz_select(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP2]], 32
-; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
-; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP3]]
-; CHECK-NEXT: ret i32 [[RETVAL]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
%conv1 = trunc i64 %val to i32
@@ -123,15 +109,9 @@ entry:
define i32 @split_cttz_select_or(i64 noundef %val) {
; CHECK-LABEL: @split_cttz_select_or(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = or disjoint i32 [[TMP2]], 32
-; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
-; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP3]]
-; CHECK-NEXT: ret i32 [[RETVAL]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
%conv1 = trunc i64 %val to i32
@@ -149,15 +129,9 @@ entry:
define i32 @split_cttz_select_ne(i64 noundef %val) {
; CHECK-LABEL: @split_cttz_select_ne(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[TMP1]], 0
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP2]], 32
-; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
-; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[TMP3]], i32 [[ADD]]
-; CHECK-NEXT: ret i32 [[RETVAL]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
%conv1 = trunc i64 %val to i32
@@ -175,15 +149,9 @@ entry:
define i32 @split_cttz_select_shift(i64 noundef %val) {
; CHECK-LABEL: @split_cttz_select_shift(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0:%.*]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP2]], 32
-; CHECK-NEXT: [[TMP3:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
-; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP3]]
-; CHECK-NEXT: [[RESULT:%.*]] = lshr i32 [[RETVAL]], 3
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT: [[RESULT:%.*]] = lshr i32 [[TMP1]], 3
; CHECK-NEXT: ret i32 [[RESULT]]
;
entry:
>From ec035ccebf3ff153108537a4519e4a935e5d1a88 Mon Sep 17 00:00:00 2001
From: Usha Gupta <usha.gupta at arm.com>
Date: Thu, 16 Apr 2026 11:13:54 +0000
Subject: [PATCH 3/6] Address review comments: generalized the fold for integer
types, removed dominance check and modified to match on or disjoint
---
.../AggressiveInstCombine.cpp | 299 +++++++++---------
.../AggressiveInstCombine/fold-split-ctlz.ll | 130 ++++++++
.../AggressiveInstCombine/fold-split-cttz.ll | 131 ++++++++
3 files changed, 418 insertions(+), 142 deletions(-)
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index bb5cacf43ad9f..06dba966cca4c 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -76,43 +76,48 @@ static cl::opt<unsigned>
cl::desc("The maximum length of a constant string to "
"inline a memchr call."));
-/// Try to fold a phi that merges two 32-bit cttz operations into a single
-/// 64-bit cttz. This pattern arises when a 64-bit cttz is implemented as
-/// two 32-bit cttz operations with a branch on whether the low half is zero:
+/// Try to fold a phi that merges two half-width cttz operations into a single
+/// full-width cttz. This pattern arises when a wide cttz is implemented as
+/// two half-width cttz operations with a branch on whether the low half is
+/// zero:
///
/// GuardBB:
-/// %lo = trunc i64 %val to i32
-/// %cmp = icmp eq i32 %lo, 0
+/// %lo = trunc iN %val to i(N/2)
+/// %cmp = icmp eq i(N/2) %lo, 0
/// br i1 %cmp, label %HiBB, label %LoBB
///
/// HiBB:
-/// %shr = lshr i64 %val, 32
-/// %hi = trunc i64 %shr to i32
-/// %cttz_hi = call i32 @llvm.cttz.i32(i32 %hi, i1 false)
-/// %val_is_zero = icmp eq i64 %val, 0
-/// %cttz_hi_plus32 = or i32 %cttz_hi, 32
-/// %hi_result = select i1 %val_is_zero, i32 ZeroResult, i32 %cttz_hi_plus32
+/// %shr = lshr iN %val, N/2
+/// %hi = trunc iN %shr to i(N/2)
+/// %cttz_hi = call i(N/2) @llvm.cttz.i(N/2)(i(N/2) %hi, i1 false)
+/// %val_is_zero = icmp eq iN %val, 0
+/// %cttz_hi_plus = add/or_disjoint i(N/2) %cttz_hi, N/2
+/// %hi_result = select i1 %val_is_zero, i(N/2) ZeroResult, i(N/2)
+/// %cttz_hi_plus
/// br label %PhiBB
///
/// LoBB:
-/// %cttz_lo = call i32 @llvm.cttz.i32(i32 %lo, i1 true)
+/// %cttz_lo = call i(N/2) @llvm.cttz.i(N/2)(i(N/2) %lo, i1 true)
/// br label %PhiBB
///
/// PhiBB:
-/// %result = phi i32 [%hi_result, %HiBB], [%cttz_lo, %LoBB]
+/// %result = phi i(N/2) [%hi_result, %HiBB], [%cttz_lo, %LoBB]
/// -->
-/// %cttz64 = call i64 @llvm.cttz.i64(i64 %val, i1 true)
-/// %trunc = trunc i64 %cttz64 to i32
-/// %val_is_zero = icmp eq i64 %val, 0
-/// %result = select i1 %val_is_zero, i32 ZeroResult, i32 %trunc
+/// %cttz_wide = call iN @llvm.cttz.iN(iN %val, i1 true)
+/// %trunc = trunc iN %cttz_wide to i(N/2)
+/// %val_is_zero = icmp eq iN %val, 0
+/// %result = select i1 %val_is_zero, i(N/2) ZeroResult, i(N/2) %trunc
///
-/// Alive proof: https://alive2.llvm.org/ce/z/FHgJpq
-static bool foldSplitWidthCTTZ(Instruction &I, const DominatorTree &DT) {
+/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/FHgJpq
+static bool foldSplitWidthCTTZ(Instruction &I) {
if (I.getOpcode() != Instruction::PHI || I.getNumOperands() != 2)
return false;
- if (!I.getType()->isIntegerTy(32))
+ Type *HalfTy = I.getType();
+ if (!HalfTy->isIntegerTy())
return false;
+ unsigned HalfWidth = HalfTy->getIntegerBitWidth();
+ unsigned FullWidth = HalfWidth * 2;
PHINode &Phi = cast<PHINode>(I);
@@ -124,22 +129,21 @@ static bool foldSplitWidthCTTZ(Instruction &I, const DominatorTree &DT) {
BasicBlock *HiBB = Phi.getIncomingBlock(HiIdx);
BasicBlock *LoBB = Phi.getIncomingBlock(LoIdx);
- // Match LoVal: cttz.i32(trunc(SrcVal to i32), _)
+ // Match LoVal: cttz.i(N/2)(trunc(SrcVal to i(N/2)), _)
Value *LoTrunc;
if (!match(LoVal,
m_Intrinsic<Intrinsic::cttz>(m_Value(LoTrunc), m_Value())))
continue;
- // LoTrunc must be a trunc of an i64 value.
+ // LoTrunc must be a trunc of a value with double the width.
Value *SrcVal = nullptr;
if (!match(LoTrunc, m_Trunc(m_Value(SrcVal))))
continue;
- if (!SrcVal->getType()->isIntegerTy(64))
+ if (!SrcVal->getType()->isIntegerTy(FullWidth))
continue;
- // Match HiVal: select(icmp eq SrcVal, 0, C,
- // or(cttz.i32(trunc(lshr(SrcVal, 32)), _), 32))
- // where C is the result when the full value is zero.
+ // Match HiVal: select(icmp eq SrcVal, 0, ZeroResult,
+ // add/or_disjoint(cttz(trunc(lshr(SrcVal, N/2))), N/2))
Value *SelectCond, *SelectTrue, *SelectFalse;
if (!match(HiVal, m_Select(m_Value(SelectCond), m_Value(SelectTrue),
m_Value(SelectFalse))))
@@ -155,52 +159,49 @@ static bool foldSplitWidthCTTZ(Instruction &I, const DominatorTree &DT) {
if (!match(SelectTrue, m_APInt(ZeroResult)))
continue;
- // SelectFalse: or(cttz.i32(trunc(lshr(SrcVal, 32)), _), 32)
- // Also accept add instead of or.
+ // SelectFalse: add/or_disjoint(cttz(trunc(lshr(SrcVal, N/2))), N/2)
Value *CttzHiCall;
- if (!match(SelectFalse, m_c_Or(m_Value(CttzHiCall), m_SpecificInt(32))) &&
- !match(SelectFalse, m_c_Add(m_Value(CttzHiCall), m_SpecificInt(32))))
+ if (!match(SelectFalse,
+ m_c_Add(m_Value(CttzHiCall), m_SpecificInt(HalfWidth))) &&
+ !match(SelectFalse,
+ m_c_DisjointOr(m_Value(CttzHiCall), m_SpecificInt(HalfWidth))))
continue;
- // CttzHiCall: cttz.i32(trunc(lshr(SrcVal, 32)), _)
+ // CttzHiCall: cttz(trunc(lshr(SrcVal, N/2)), _)
Value *HiTrunc;
if (!match(CttzHiCall,
m_Intrinsic<Intrinsic::cttz>(m_Value(HiTrunc), m_Value())))
continue;
- if (!match(HiTrunc, m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(32)))))
+ if (!match(HiTrunc,
+ m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(HalfWidth)))))
continue;
// Both HiBB and LoBB share a single predecessor
- // that branches on whether the low 32 bits are zero.
+ // that branches on whether the low half is zero.
BasicBlock *GuardBB = LoBB->getSinglePredecessor();
if (!GuardBB || GuardBB != HiBB->getSinglePredecessor())
continue;
Instruction *GuardTerm = GuardBB->getTerminator();
- // Guard: br (icmp eq (trunc SrcVal to i32), 0), HiBB, LoBB
+ // Guard: br (icmp eq (trunc SrcVal to i(N/2)), 0), HiBB, LoBB
if (!match(GuardTerm,
m_Br(m_SpecificICmp(CmpInst::ICMP_EQ,
m_Trunc(m_Specific(SrcVal)), m_ZeroInt()),
m_SpecificBB(HiBB), m_SpecificBB(LoBB))))
continue;
- // Verify SrcVal dominates the guard.
- if (auto *SrcI = dyn_cast<Instruction>(SrcVal))
- if (!DT.dominates(SrcI, GuardTerm))
- continue;
-
// Match successful.
BasicBlock *PhiBB = Phi.getParent();
IRBuilder<> Builder(PhiBB, PhiBB->getFirstInsertionPt());
- Value *Cttz64 = Builder.CreateIntrinsic(
+ Value *CttzWide = Builder.CreateIntrinsic(
Intrinsic::cttz, {SrcVal->getType()}, {SrcVal, Builder.getTrue()});
- Value *Trunc = Builder.CreateTrunc(Cttz64, Builder.getInt32Ty());
+ Value *Trunc = Builder.CreateTrunc(CttzWide, HalfTy);
Value *IsZero =
Builder.CreateICmpEQ(SrcVal, ConstantInt::get(SrcVal->getType(), 0));
Value *Result = Builder.CreateSelect(
- IsZero, ConstantInt::get(Builder.getInt32Ty(), *ZeroResult), Trunc);
+ IsZero, ConstantInt::get(HalfTy, *ZeroResult), Trunc);
Phi.replaceAllUsesWith(Result);
++NumSplitCTTZFolded;
@@ -210,42 +211,47 @@ static bool foldSplitWidthCTTZ(Instruction &I, const DominatorTree &DT) {
return false;
}
-/// Try to fold a phi that merges two 32-bit ctlz operations into a single
-/// 64-bit ctlz. This is similar to foldSplitWidthCTTZ, but for leading zeros.
+/// Try to fold a phi that merges two half-width ctlz operations into a single
+/// full-width ctlz. This is similar to foldSplitWidthCTTZ, but for leading
+/// zeros.
///
/// GuardBB:
-/// %shr = lshr i64 %val, 32
-/// %cmp = icmp eq i64 %shr, 0
+/// %shr = lshr iN %val, N/2
+/// %cmp = icmp eq iN %shr, 0
/// br i1 %cmp, label %LoBB, label %HiBB
///
-/// LoBB (upper is zero: count in lower half + 32):
-/// %lo = trunc i64 %val to i32
-/// %ctlz_lo = call i32 @llvm.ctlz.i32(i32 %lo, i1 true)
-/// %val_is_zero = icmp eq i64 %val, 0
-/// %ctlz_lo_plus32 = or i32 %ctlz_lo, 32
-/// %lo_result = select i1 %val_is_zero, i32 ZeroResult, i32 %ctlz_lo_plus32
+/// LoBB (upper is zero: count in lower half + N/2):
+/// %lo = trunc iN %val to i(N/2)
+/// %ctlz_lo = call i(N/2) @llvm.ctlz.i(N/2)(i(N/2) %lo, i1 true)
+/// %val_is_zero = icmp eq iN %val, 0
+/// %ctlz_lo_plus = add/or_disjoint i(N/2) %ctlz_lo, N/2
+/// %lo_result = select i1 %val_is_zero, i(N/2) ZeroResult, i(N/2)
+/// %ctlz_lo_plus
/// br label %PhiBB
///
/// HiBB (upper is non-zero: count in upper half):
-/// %hi = trunc i64 %shr to i32
-/// %ctlz_hi = call i32 @llvm.ctlz.i32(i32 %hi, i1 true)
+/// %hi = trunc iN %shr to i(N/2)
+/// %ctlz_hi = call i(N/2) @llvm.ctlz.i(N/2)(i(N/2) %hi, i1 true)
/// br label %PhiBB
///
/// PhiBB:
-/// %result = phi i32 [%lo_result, %LoBB], [%ctlz_hi, %HiBB]
+/// %result = phi i(N/2) [%lo_result, %LoBB], [%ctlz_hi, %HiBB]
/// -->
-/// %ctlz64 = call i64 @llvm.ctlz.i64(i64 %val, i1 true)
-/// %trunc = trunc i64 %ctlz64 to i32
-/// %val_is_zero = icmp eq i64 %val, 0
-/// %result = select i1 %val_is_zero, i32 ZeroResult, i32 %trunc
+/// %ctlz_wide = call iN @llvm.ctlz.iN(iN %val, i1 true)
+/// %trunc = trunc iN %ctlz_wide to i(N/2)
+/// %val_is_zero = icmp eq iN %val, 0
+/// %result = select i1 %val_is_zero, i(N/2) ZeroResult, i(N/2) %trunc
///
-/// Alive proof: https://alive2.llvm.org/ce/z/peel2Y
-static bool foldSplitWidthCTLZ(Instruction &I, const DominatorTree &DT) {
+/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/peel2Y
+static bool foldSplitWidthCTLZ(Instruction &I) {
if (I.getOpcode() != Instruction::PHI || I.getNumOperands() != 2)
return false;
- if (!I.getType()->isIntegerTy(32))
+ Type *HalfTy = I.getType();
+ if (!HalfTy->isIntegerTy())
return false;
+ unsigned HalfWidth = HalfTy->getIntegerBitWidth();
+ unsigned FullWidth = HalfWidth * 2;
PHINode &Phi = cast<PHINode>(I);
@@ -257,22 +263,23 @@ static bool foldSplitWidthCTLZ(Instruction &I, const DominatorTree &DT) {
BasicBlock *LoBB = Phi.getIncomingBlock(LoIdx);
BasicBlock *HiBB = Phi.getIncomingBlock(HiIdx);
- // Match HiVal: ctlz.i32(trunc(lshr(SrcVal, 32) to i32), _)
- // This is the ctlz on the upper 32 bits (when upper is non-zero).
+ // Match HiVal: ctlz(trunc(lshr(SrcVal, N/2)), _)
+ // This is the ctlz on the upper half (when upper is non-zero).
Value *HiTrunc;
if (!match(HiVal,
m_Intrinsic<Intrinsic::ctlz>(m_Value(HiTrunc), m_Value())))
continue;
- // HiTrunc must be trunc(lshr(SrcVal, 32)).
+ // HiTrunc must be trunc(lshr(SrcVal, N/2)).
Value *SrcVal = nullptr;
- if (!match(HiTrunc, m_Trunc(m_LShr(m_Value(SrcVal), m_SpecificInt(32)))))
+ if (!match(HiTrunc,
+ m_Trunc(m_LShr(m_Value(SrcVal), m_SpecificInt(HalfWidth)))))
continue;
- if (!SrcVal->getType()->isIntegerTy(64))
+ if (!SrcVal->getType()->isIntegerTy(FullWidth))
continue;
// Match LoVal: select(icmp eq SrcVal, 0, ZeroResult,
- // or/add(ctlz.i32(trunc(SrcVal to i32), _), 32))
+ // add/or_disjoint(ctlz(trunc(SrcVal), _), N/2))
Value *SelectCond, *SelectTrue, *SelectFalse;
if (!match(LoVal, m_Select(m_Value(SelectCond), m_Value(SelectTrue),
m_Value(SelectFalse))))
@@ -288,13 +295,15 @@ static bool foldSplitWidthCTLZ(Instruction &I, const DominatorTree &DT) {
if (!match(SelectTrue, m_APInt(ZeroResult)))
continue;
- // SelectFalse: or/add(ctlz.i32(trunc(SrcVal to i32), _), 32)
+ // SelectFalse: add/or_disjoint(ctlz(trunc(SrcVal), _), N/2)
Value *CtlzLoCall;
- if (!match(SelectFalse, m_c_Or(m_Value(CtlzLoCall), m_SpecificInt(32))) &&
- !match(SelectFalse, m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(32))))
+ if (!match(SelectFalse,
+ m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(HalfWidth))) &&
+ !match(SelectFalse,
+ m_c_DisjointOr(m_Value(CtlzLoCall), m_SpecificInt(HalfWidth))))
continue;
- // CtlzLoCall: ctlz.i32(trunc(SrcVal to i32), _)
+ // CtlzLoCall: ctlz(trunc(SrcVal), _)
Value *LoTrunc;
if (!match(CtlzLoCall,
m_Intrinsic<Intrinsic::ctlz>(m_Value(LoTrunc), m_Value())))
@@ -304,36 +313,31 @@ static bool foldSplitWidthCTLZ(Instruction &I, const DominatorTree &DT) {
continue;
// Both LoBB and HiBB share a single predecessor that branches on whether
- // the upper 32 bits are zero.
+ // the upper half is zero.
BasicBlock *GuardBB = LoBB->getSinglePredecessor();
if (!GuardBB || GuardBB != HiBB->getSinglePredecessor())
continue;
Instruction *GuardTerm = GuardBB->getTerminator();
- // Guard: br (icmp eq (lshr SrcVal, 32), 0), LoBB, HiBB
+ // Guard: br (icmp eq (lshr SrcVal, N/2), 0), LoBB, HiBB
if (!match(GuardTerm, m_Br(m_SpecificICmp(CmpInst::ICMP_EQ,
m_LShr(m_Specific(SrcVal),
- m_SpecificInt(32)),
+ m_SpecificInt(HalfWidth)),
m_ZeroInt()),
m_SpecificBB(LoBB), m_SpecificBB(HiBB))))
continue;
- // Verify SrcVal dominates the guard.
- if (auto *SrcI = dyn_cast<Instruction>(SrcVal))
- if (!DT.dominates(SrcI, GuardTerm))
- continue;
-
// Match successful.
BasicBlock *PhiBB = Phi.getParent();
IRBuilder<> Builder(PhiBB, PhiBB->getFirstInsertionPt());
- Value *Ctlz64 = Builder.CreateIntrinsic(
+ Value *CtlzWide = Builder.CreateIntrinsic(
Intrinsic::ctlz, {SrcVal->getType()}, {SrcVal, Builder.getTrue()});
- Value *Trunc = Builder.CreateTrunc(Ctlz64, Builder.getInt32Ty());
+ Value *Trunc = Builder.CreateTrunc(CtlzWide, HalfTy);
Value *IsZero =
Builder.CreateICmpEQ(SrcVal, ConstantInt::get(SrcVal->getType(), 0));
Value *Result = Builder.CreateSelect(
- IsZero, ConstantInt::get(Builder.getInt32Ty(), *ZeroResult), Trunc);
+ IsZero, ConstantInt::get(HalfTy, *ZeroResult), Trunc);
Phi.replaceAllUsesWith(Result);
++NumSplitCTLZFolded;
@@ -343,30 +347,32 @@ static bool foldSplitWidthCTLZ(Instruction &I, const DominatorTree &DT) {
return false;
}
-/// Try to fold a select-based split cttz(32-bit) pattern into a single 64-bit
-/// cttz.
+/// Try to fold a select-based split cttz pattern into a single full-width cttz.
///
-/// %lo = trunc i64 %val to i32
-/// %cmp = icmp eq i32 %lo, 0
-/// %shr = lshr i64 %val, 32
-/// %hi = trunc i64 %shr to i32
-/// %cttz_hi = call i32 @llvm.cttz.i32(i32 %hi, ...)
-/// %hi_plus32 = add/or i32 %cttz_hi, 32
-/// %cttz_lo = call i32 @llvm.cttz.i32(i32 %lo, ...)
-/// %result = select i1 %cmp, i32 %hi_plus32, i32 %cttz_lo
+/// %lo = trunc iN %val to i(N/2)
+/// %cmp = icmp eq i(N/2) %lo, 0
+/// %shr = lshr iN %val, N/2
+/// %hi = trunc iN %shr to i(N/2)
+/// %cttz_hi = call i(N/2) @llvm.cttz.i(N/2)(i(N/2) %hi, ...)
+/// %hi_plus = add/or_disjoint i(N/2) %cttz_hi, N/2
+/// %cttz_lo = call i(N/2) @llvm.cttz.i(N/2)(i(N/2) %lo, ...)
+/// %result = select i1 %cmp, i(N/2) %hi_plus, i(N/2) %cttz_lo
/// -->
-/// %cttz64 = call i64 @llvm.cttz.i64(i64 %val, i1 false)
-/// %result = trunc i64 %cttz64 to i32
-/// Alive prrof: https://alive2.llvm.org/ce/z/2MkVVM
+/// %cttz_wide = call iN @llvm.cttz.iN(iN %val, i1 false)
+/// %result = trunc iN %cttz_wide to i(N/2)
+/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/2MkVVM
static bool foldSelectSplitCTTZ(Instruction &I) {
Value *Cond, *TrueVal, *FalseVal;
if (!match(&I, m_Select(m_Value(Cond), m_Value(TrueVal), m_Value(FalseVal))))
return false;
- if (!I.getType()->isIntegerTy(32))
+ Type *HalfTy = I.getType();
+ if (!HalfTy->isIntegerTy())
return false;
+ unsigned HalfWidth = HalfTy->getIntegerBitWidth();
+ unsigned FullWidth = HalfWidth * 2;
- // select (icmp eq (trunc SrcVal to i32), 0), HiResult, LoResult
+ // select (icmp eq (trunc SrcVal to i(N/2)), 0), HiResult, LoResult
// Or select (icmp ne ...), LoResult, HiResult
Value *LoTrunc;
Value *HiResult, *LoResult;
@@ -382,14 +388,14 @@ static bool foldSelectSplitCTTZ(Instruction &I) {
return false;
}
- // LoTrunc: trunc i64 SrcVal to i32
+ // LoTrunc: trunc iN SrcVal to i(N/2)
Value *SrcVal;
if (!match(LoTrunc, m_Trunc(m_Value(SrcVal))))
return false;
- if (!SrcVal->getType()->isIntegerTy(64))
+ if (!SrcVal->getType()->isIntegerTy(FullWidth))
return false;
- // LoResult: cttz.i32(trunc(SrcVal), _), must use same truncated value
+ // LoResult: cttz(trunc(SrcVal), _), must use same truncated value
Value *LoCttzArg;
if (!match(LoResult,
m_Intrinsic<Intrinsic::cttz>(m_Value(LoCttzArg), m_Value())))
@@ -397,10 +403,12 @@ static bool foldSelectSplitCTTZ(Instruction &I) {
if (LoCttzArg != LoTrunc)
return false;
- // HiResult: add/or(cttz.i32(trunc(lshr(SrcVal, 32)), _), 32)
+ // HiResult: add/or_disjoint(cttz(trunc(lshr(SrcVal, N/2)), _), N/2)
Value *CttzHiCall;
- if (!match(HiResult, m_c_Add(m_Value(CttzHiCall), m_SpecificInt(32))) &&
- !match(HiResult, m_c_Or(m_Value(CttzHiCall), m_SpecificInt(32))))
+ if (!match(HiResult,
+ m_c_Add(m_Value(CttzHiCall), m_SpecificInt(HalfWidth))) &&
+ !match(HiResult,
+ m_c_DisjointOr(m_Value(CttzHiCall), m_SpecificInt(HalfWidth))))
return false;
Value *HiCttzArg;
@@ -408,14 +416,15 @@ static bool foldSelectSplitCTTZ(Instruction &I) {
m_Intrinsic<Intrinsic::cttz>(m_Value(HiCttzArg), m_Value())))
return false;
- if (!match(HiCttzArg, m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(32)))))
+ if (!match(HiCttzArg,
+ m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(HalfWidth)))))
return false;
// Match successful.
IRBuilder<> Builder(&I);
- Value *Cttz64 = Builder.CreateIntrinsic(Intrinsic::cttz, {SrcVal->getType()},
- {SrcVal, Builder.getFalse()});
- Value *Trunc = Builder.CreateTrunc(Cttz64, Builder.getInt32Ty());
+ Value *CttzWide = Builder.CreateIntrinsic(
+ Intrinsic::cttz, {SrcVal->getType()}, {SrcVal, Builder.getFalse()});
+ Value *Trunc = Builder.CreateTrunc(CttzWide, HalfTy);
I.replaceAllUsesWith(Trunc);
++NumSelectCTTZFolded;
@@ -424,35 +433,38 @@ static bool foldSelectSplitCTTZ(Instruction &I) {
/// Same as foldSelectSplitCTTZ but for leading zeros (ctlz).
///
-/// %shr = lshr i64 %val, 32
-/// %hi = trunc i64 %shr to i32
-/// %cmp = icmp eq i32 %hi, 0 (or icmp eq i64 %shr, 0)
-/// %lo = trunc i64 %val to i32
-/// %ctlz_lo = call i32 @llvm.ctlz.i32(i32 %lo, ...)
-/// %lo_plus32 = add/or i32 %ctlz_lo, 32
-/// %ctlz_hi = call i32 @llvm.ctlz.i32(i32 %hi, ...)
-/// %result = select i1 %cmp, i32 %lo_plus32, i32 %ctlz_hi
+/// %shr = lshr iN %val, N/2
+/// %hi = trunc iN %shr to i(N/2)
+/// %cmp = icmp eq i(N/2) %hi, 0 (or icmp eq iN %shr, 0)
+/// %lo = trunc iN %val to i(N/2)
+/// %ctlz_lo = call i(N/2) @llvm.ctlz.i(N/2)(i(N/2) %lo, ...)
+/// %lo_plus = add/or_disjoint i(N/2) %ctlz_lo, N/2
+/// %ctlz_hi = call i(N/2) @llvm.ctlz.i(N/2)(i(N/2) %hi, ...)
+/// %result = select i1 %cmp, i(N/2) %lo_plus, i(N/2) %ctlz_hi
/// -->
-/// %ctlz64 = call i64 @llvm.ctlz.i64(i64 %val, i1 false)
-/// %result = trunc i64 %ctlz64 to i32
+/// %ctlz_wide = call iN @llvm.ctlz.iN(iN %val, i1 false)
+/// %result = trunc iN %ctlz_wide to i(N/2)
///
-/// Alive proof: https://alive2.llvm.org/ce/z/rBJygf
+/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/rBJygf
static bool foldSelectSplitCTLZ(Instruction &I) {
Value *Cond, *TrueVal, *FalseVal;
if (!match(&I, m_Select(m_Value(Cond), m_Value(TrueVal), m_Value(FalseVal))))
return false;
- if (!I.getType()->isIntegerTy(32))
+ Type *HalfTy = I.getType();
+ if (!HalfTy->isIntegerTy())
return false;
+ unsigned HalfWidth = HalfTy->getIntegerBitWidth();
+ unsigned FullWidth = HalfWidth * 2;
// select (icmp eq HiPart, 0), LoResult, HiResult
- // HiPart could be (trunc (lshr SrcVal, 32) to i32) or (lshr SrcVal, 32)
+ // HiPart could be (trunc (lshr SrcVal, N/2) to i(N/2)) or (lshr SrcVal, N/2)
Value *HiPart;
Value *LoResult, *HiResult;
if (match(Cond,
m_SpecificICmp(CmpInst::ICMP_EQ, m_Value(HiPart), m_ZeroInt()))) {
- LoResult = TrueVal; // upper is zero
- HiResult = FalseVal; // upper non-zero
+ LoResult = TrueVal; // upper is zero: count in lower + N/2
+ HiResult = FalseVal; // upper non-zero: count in upper
} else if (match(Cond, m_SpecificICmp(CmpInst::ICMP_NE, m_Value(HiPart),
m_ZeroInt()))) {
LoResult = FalseVal;
@@ -461,33 +473,36 @@ static bool foldSelectSplitCTLZ(Instruction &I) {
return false;
}
- // Extract SrcVal from HiPart: either trunc(lshr(SrcVal, 32)) or
- // lshr(SrcVal, 32)
+ // Extract SrcVal from HiPart: either trunc(lshr(SrcVal, N/2)) or
+ // lshr(SrcVal, N/2)
Value *SrcVal;
- if (match(HiPart, m_Trunc(m_LShr(m_Value(SrcVal), m_SpecificInt(32))))) {
- // HiPart is trunc(lshr(SrcVal, 32))
- } else if (match(HiPart, m_LShr(m_Value(SrcVal), m_SpecificInt(32)))) {
- // HiPart is lshr(SrcVal, 32)
+ if (match(HiPart,
+ m_Trunc(m_LShr(m_Value(SrcVal), m_SpecificInt(HalfWidth))))) {
+ // HiPart is trunc(lshr(SrcVal, N/2))
+ } else if (match(HiPart, m_LShr(m_Value(SrcVal), m_SpecificInt(HalfWidth)))) {
+ // HiPart is lshr(SrcVal, N/2)
} else {
return false;
}
- if (!SrcVal->getType()->isIntegerTy(64))
+ if (!SrcVal->getType()->isIntegerTy(FullWidth))
return false;
- // HiResult: ctlz.i32(trunc(lshr(SrcVal,32)), _)
+ // HiResult: ctlz(trunc(lshr(SrcVal, N/2)), _)
Value *HiCtlzArg;
if (!match(HiResult,
m_Intrinsic<Intrinsic::ctlz>(m_Value(HiCtlzArg), m_Value())))
return false;
- // HiCtlzArg should be a trunc of lshr(SrcVal, 32)
- if (!match(HiCtlzArg, m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(32)))))
+ if (!match(HiCtlzArg,
+ m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(HalfWidth)))))
return false;
- // LoResult: add/or(ctlz.i32(trunc(SrcVal), _), 32)
+ // LoResult: add/or_disjoint(ctlz(trunc(SrcVal), _), N/2)
Value *CtlzLoCall;
- if (!match(LoResult, m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(32))) &&
- !match(LoResult, m_c_Or(m_Value(CtlzLoCall), m_SpecificInt(32))))
+ if (!match(LoResult,
+ m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(HalfWidth))) &&
+ !match(LoResult,
+ m_c_DisjointOr(m_Value(CtlzLoCall), m_SpecificInt(HalfWidth))))
return false;
Value *LoCtlzArg;
@@ -498,11 +513,11 @@ static bool foldSelectSplitCTLZ(Instruction &I) {
if (!match(LoCtlzArg, m_Trunc(m_Specific(SrcVal))))
return false;
- // Match successsful.
+ // Match successful.
IRBuilder<> Builder(&I);
- Value *Ctlz64 = Builder.CreateIntrinsic(Intrinsic::ctlz, {SrcVal->getType()},
- {SrcVal, Builder.getFalse()});
- Value *Trunc = Builder.CreateTrunc(Ctlz64, Builder.getInt32Ty());
+ Value *CtlzWide = Builder.CreateIntrinsic(
+ Intrinsic::ctlz, {SrcVal->getType()}, {SrcVal, Builder.getFalse()});
+ Value *Trunc = Builder.CreateTrunc(CtlzWide, HalfTy);
I.replaceAllUsesWith(Trunc);
++NumSelectCTLZFolded;
@@ -2464,8 +2479,8 @@ static bool foldUnusualPatterns(Function &F, DominatorTree &DT,
for (Instruction &I : make_early_inc_range(llvm::reverse(BB))) {
MadeChange |= foldAnyOrAllBitsSet(I);
MadeChange |= foldGuardedFunnelShift(I, DT);
- MadeChange |= foldSplitWidthCTTZ(I, DT);
- MadeChange |= foldSplitWidthCTLZ(I, DT);
+ MadeChange |= foldSplitWidthCTTZ(I);
+ MadeChange |= foldSplitWidthCTLZ(I);
MadeChange |= foldSelectSplitCTTZ(I);
MadeChange |= foldSelectSplitCTLZ(I);
MadeChange |= tryToRecognizePopCount(I);
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
index db5296dcf05ca..df6ef312d27c1 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
@@ -164,6 +164,107 @@ entry:
ret i32 %retval
}
+; i32 split into two i16 ctlz (phi-based).
+define i16 @split_ctlz_phi_i32(i32 noundef %val) {
+; CHECK-LABEL: @split_ctlz_phi_i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[VAL:%.*]], 16
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[SHR]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: br label [[CLEANUP:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: br label [[CLEANUP]]
+; CHECK: cleanup:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[VAL]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i16
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[VAL]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i16 16, i16 [[TMP1]]
+; CHECK-NEXT: ret i16 [[TMP3]]
+;
+entry:
+ %shr = lshr i32 %val, 16
+ %cmp = icmp eq i32 %shr, 0
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+ %conv_lo = trunc i32 %val to i16
+ %0 = call i16 @llvm.ctlz.i16(i16 %conv_lo, i1 true)
+ %1 = icmp eq i32 %val, 0
+ %2 = add i16 %0, 16
+ %add = select i1 %1, i16 16, i16 %2
+ br label %cleanup
+
+if.else:
+ %conv_hi = trunc nuw i32 %shr to i16
+ %3 = call i16 @llvm.ctlz.i16(i16 %conv_hi, i1 true)
+ br label %cleanup
+
+cleanup:
+ %retval.0 = phi i16 [ %add, %if.then ], [ %3, %if.else ]
+ ret i16 %retval.0
+}
+
+; i32 split into two i16 ctlz (select-based).
+define i16 @split_ctlz_select_i32(i32 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i16
+; CHECK-NEXT: ret i16 [[TMP1]]
+;
+entry:
+ %shr = lshr i32 %val, 16
+ %conv_hi = trunc nuw i32 %shr to i16
+ %cmp = icmp eq i16 %conv_hi, 0
+ %conv_lo = trunc i32 %val to i16
+ %0 = call i16 @llvm.ctlz.i16(i16 %conv_lo, i1 false)
+ %add = add nuw nsw i16 %0, 16
+ %1 = call i16 @llvm.ctlz.i16(i16 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i16 %add, i16 %1
+ ret i16 %retval
+}
+
+; i128 split into two i64 ctlz (select-based).
+define i64 @split_ctlz_select_i128(i128 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_i128(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i128 @llvm.ctlz.i128(i128 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i128 [[TMP0]] to i64
+; CHECK-NEXT: ret i64 [[TMP1]]
+;
+entry:
+ %shr = lshr i128 %val, 64
+ %conv_hi = trunc nuw i128 %shr to i64
+ %cmp = icmp eq i64 %conv_hi, 0
+ %conv_lo = trunc i128 %val to i64
+ %0 = call i64 @llvm.ctlz.i64(i64 %conv_lo, i1 false)
+ %add = add nuw nsw i64 %0, 64
+ %1 = call i64 @llvm.ctlz.i64(i64 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i64 %add, i64 %1
+ ret i64 %retval
+}
+
+; i16 split into two i8 ctlz (select-based).
+define i8 @split_ctlz_select_i16(i16 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_i16(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i16 @llvm.ctlz.i16(i16 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i16 [[TMP0]] to i8
+; CHECK-NEXT: ret i8 [[TMP1]]
+;
+entry:
+ %shr = lshr i16 %val, 8
+ %conv_hi = trunc nuw i16 %shr to i8
+ %cmp = icmp eq i8 %conv_hi, 0
+ %conv_lo = trunc i16 %val to i8
+ %0 = call i8 @llvm.ctlz.i8(i8 %conv_lo, i1 false)
+ %add = add nuw nsw i8 %0, 8
+ %1 = call i8 @llvm.ctlz.i8(i8 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i8 %add, i8 %1
+ ret i8 %retval
+}
+
; Negative test: mismatched source values, should NOT be folded.
define i32 @split_ctlz_different_sources(i64 noundef %val, i64 noundef %val2) {
; CHECK-LABEL: @split_ctlz_different_sources(
@@ -216,5 +317,34 @@ entry:
ret i32 %retval
}
+; Negative test: plain or (without disjoint flag), should NOT be folded
+; in select-based pattern because or(32, 32)=32 != add(32, 32)=64.
+define i32 @split_ctlz_select_plain_or(i64 noundef %val) {
+; CHECK-LABEL: @split_ctlz_select_plain_or(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %conv_hi = trunc nuw i64 %shr to i32
+ %cmp = icmp eq i32 %conv_hi, 0
+ %conv_lo = trunc i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = or i32 %0, 32
+ %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+declare i8 @llvm.ctlz.i8(i8, i1 immarg)
+declare i16 @llvm.ctlz.i16(i16, i1 immarg)
declare i32 @llvm.ctlz.i32(i32, i1 immarg)
declare i64 @llvm.ctlz.i64(i64, i1 immarg)
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
index b228bf6576baf..15879ec05ab8f 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
@@ -167,6 +167,108 @@ entry:
ret i32 %result
}
+; i32 split into two i16 cttz (phi-based).
+define i16 @split_cttz_phi_i32(i32 noundef %val) {
+; CHECK-LABEL: @split_cttz_phi_i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i32 [[VAL:%.*]] to i16
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i16 [[CONV1]], 0
+; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: br label [[CLEANUP:%.*]]
+; CHECK: if.else:
+; CHECK-NEXT: br label [[CLEANUP]]
+; CHECK: cleanup:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[VAL]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i16
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[VAL]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i16 16, i16 [[TMP1]]
+; CHECK-NEXT: ret i16 [[TMP3]]
+;
+entry:
+ %conv1 = trunc i32 %val to i16
+ %cmp = icmp eq i16 %conv1, 0
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+ %shr = lshr i32 %val, 16
+ %conv = trunc nuw i32 %shr to i16
+ %0 = call i16 @llvm.cttz.i16(i16 %conv, i1 true)
+ %1 = icmp eq i32 %val, 0
+ %2 = add i16 %0, 16
+ %add = select i1 %1, i16 16, i16 %2
+ br label %cleanup
+
+if.else:
+ %3 = call i16 @llvm.cttz.i16(i16 %conv1, i1 true)
+ br label %cleanup
+
+cleanup:
+ %retval.0 = phi i16 [ %add, %if.then ], [ %3, %if.else ]
+ ret i16 %retval.0
+}
+
+; i32 split into two i16 cttz (select-based).
+define i16 @split_cttz_select_i32(i32 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i16
+; CHECK-NEXT: ret i16 [[TMP1]]
+;
+entry:
+ %conv1 = trunc i32 %val to i16
+ %cmp = icmp eq i16 %conv1, 0
+ %shr = lshr i32 %val, 16
+ %conv = trunc nuw i32 %shr to i16
+ %0 = call i16 @llvm.cttz.i16(i16 %conv, i1 false)
+ %add = add nuw nsw i16 %0, 16
+ %1 = call i16 @llvm.cttz.i16(i16 %conv1, i1 true)
+ %retval = select i1 %cmp, i16 %add, i16 %1
+ ret i16 %retval
+}
+
+; i128 split into two i64 cttz (select-based).
+define i64 @split_cttz_select_i128(i128 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_i128(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i128 @llvm.cttz.i128(i128 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i128 [[TMP0]] to i64
+; CHECK-NEXT: ret i64 [[TMP1]]
+;
+entry:
+ %conv1 = trunc i128 %val to i64
+ %cmp = icmp eq i64 %conv1, 0
+ %shr = lshr i128 %val, 64
+ %conv = trunc nuw i128 %shr to i64
+ %0 = call i64 @llvm.cttz.i64(i64 %conv, i1 false)
+ %add = add nuw nsw i64 %0, 64
+ %1 = call i64 @llvm.cttz.i64(i64 %conv1, i1 true)
+ %retval = select i1 %cmp, i64 %add, i64 %1
+ ret i64 %retval
+}
+
+; i16 split into two i8 cttz (select-based).
+define i8 @split_cttz_select_i16(i16 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_i16(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i16 @llvm.cttz.i16(i16 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i16 [[TMP0]] to i8
+; CHECK-NEXT: ret i8 [[TMP1]]
+;
+entry:
+ %conv1 = trunc i16 %val to i8
+ %cmp = icmp eq i8 %conv1, 0
+ %shr = lshr i16 %val, 8
+ %conv = trunc nuw i16 %shr to i8
+ %0 = call i8 @llvm.cttz.i8(i8 %conv, i1 false)
+ %add = add nuw nsw i8 %0, 8
+ %1 = call i8 @llvm.cttz.i8(i8 %conv1, i1 true)
+ %retval = select i1 %cmp, i8 %add, i8 %1
+ ret i8 %retval
+}
+
+
; Negative test: mismatched source values, should NOT be folded.
define i32 @split_cttz_different_sources(i64 noundef %val, i64 noundef %val2) {
; CHECK-LABEL: @split_cttz_different_sources(
@@ -219,5 +321,34 @@ entry:
ret i32 %retval
}
+; Negative test: plain or (without disjoint flag), should NOT be folded
+; in select-based pattern because or(32, 32)=32 != add(32, 32)=64.
+define i32 @split_cttz_select_plain_or(i64 noundef %val) {
+; CHECK-LABEL: @split_cttz_select_plain_or(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ %shr = lshr i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = or i32 %0, 32
+ %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ ret i32 %retval
+}
+
+declare i8 @llvm.cttz.i8(i8, i1 immarg)
+declare i16 @llvm.cttz.i16(i16, i1 immarg)
declare i32 @llvm.cttz.i32(i32, i1 immarg)
declare i64 @llvm.cttz.i64(i64, i1 immarg)
>From 2ec3ebc2370e84037bd046bbf73bdd1d1461dd96 Mon Sep 17 00:00:00 2001
From: Usha Gupta <usha.gupta at arm.com>
Date: Tue, 21 Apr 2026 10:26:47 +0000
Subject: [PATCH 4/6] Addressing review comments: removed phi based matcher for
now, Used m_AddLike, added m_OneUse, more tests added
---
.../AggressiveInstCombine.cpp | 309 +-----------------
.../AggressiveInstCombine/fold-split-ctlz.ll | 231 +++++--------
.../AggressiveInstCombine/fold-split-cttz.ll | 206 ++++--------
3 files changed, 154 insertions(+), 592 deletions(-)
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index 06dba966cca4c..077168878d501 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -53,10 +53,6 @@ STATISTIC(NumGuardedRotates,
STATISTIC(NumGuardedFunnelShifts,
"Number of guarded funnel shifts transformed into funnel shifts");
STATISTIC(NumPopCountRecognized, "Number of popcount idioms recognized");
-STATISTIC(NumSplitCTTZFolded,
- "Number of split-width cttz patterns folded into full-width cttz");
-STATISTIC(NumSplitCTLZFolded,
- "Number of split-width ctlz patterns folded into full-width ctlz");
STATISTIC(NumSelectCTTZFolded,
"Number of select-based split cttz patterns folded");
STATISTIC(NumSelectCTLZFolded,
@@ -76,277 +72,6 @@ static cl::opt<unsigned>
cl::desc("The maximum length of a constant string to "
"inline a memchr call."));
-/// Try to fold a phi that merges two half-width cttz operations into a single
-/// full-width cttz. This pattern arises when a wide cttz is implemented as
-/// two half-width cttz operations with a branch on whether the low half is
-/// zero:
-///
-/// GuardBB:
-/// %lo = trunc iN %val to i(N/2)
-/// %cmp = icmp eq i(N/2) %lo, 0
-/// br i1 %cmp, label %HiBB, label %LoBB
-///
-/// HiBB:
-/// %shr = lshr iN %val, N/2
-/// %hi = trunc iN %shr to i(N/2)
-/// %cttz_hi = call i(N/2) @llvm.cttz.i(N/2)(i(N/2) %hi, i1 false)
-/// %val_is_zero = icmp eq iN %val, 0
-/// %cttz_hi_plus = add/or_disjoint i(N/2) %cttz_hi, N/2
-/// %hi_result = select i1 %val_is_zero, i(N/2) ZeroResult, i(N/2)
-/// %cttz_hi_plus
-/// br label %PhiBB
-///
-/// LoBB:
-/// %cttz_lo = call i(N/2) @llvm.cttz.i(N/2)(i(N/2) %lo, i1 true)
-/// br label %PhiBB
-///
-/// PhiBB:
-/// %result = phi i(N/2) [%hi_result, %HiBB], [%cttz_lo, %LoBB]
-/// -->
-/// %cttz_wide = call iN @llvm.cttz.iN(iN %val, i1 true)
-/// %trunc = trunc iN %cttz_wide to i(N/2)
-/// %val_is_zero = icmp eq iN %val, 0
-/// %result = select i1 %val_is_zero, i(N/2) ZeroResult, i(N/2) %trunc
-///
-/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/FHgJpq
-static bool foldSplitWidthCTTZ(Instruction &I) {
- if (I.getOpcode() != Instruction::PHI || I.getNumOperands() != 2)
- return false;
-
- Type *HalfTy = I.getType();
- if (!HalfTy->isIntegerTy())
- return false;
- unsigned HalfWidth = HalfTy->getIntegerBitWidth();
- unsigned FullWidth = HalfWidth * 2;
-
- PHINode &Phi = cast<PHINode>(I);
-
- // Try both orderings of phi operands.
- for (unsigned HiIdx = 0; HiIdx < 2; ++HiIdx) {
- unsigned LoIdx = 1 - HiIdx;
- Value *HiVal = Phi.getIncomingValue(HiIdx);
- Value *LoVal = Phi.getIncomingValue(LoIdx);
- BasicBlock *HiBB = Phi.getIncomingBlock(HiIdx);
- BasicBlock *LoBB = Phi.getIncomingBlock(LoIdx);
-
- // Match LoVal: cttz.i(N/2)(trunc(SrcVal to i(N/2)), _)
- Value *LoTrunc;
- if (!match(LoVal,
- m_Intrinsic<Intrinsic::cttz>(m_Value(LoTrunc), m_Value())))
- continue;
-
- // LoTrunc must be a trunc of a value with double the width.
- Value *SrcVal = nullptr;
- if (!match(LoTrunc, m_Trunc(m_Value(SrcVal))))
- continue;
- if (!SrcVal->getType()->isIntegerTy(FullWidth))
- continue;
-
- // Match HiVal: select(icmp eq SrcVal, 0, ZeroResult,
- // add/or_disjoint(cttz(trunc(lshr(SrcVal, N/2))), N/2))
- Value *SelectCond, *SelectTrue, *SelectFalse;
- if (!match(HiVal, m_Select(m_Value(SelectCond), m_Value(SelectTrue),
- m_Value(SelectFalse))))
- continue;
-
- // SelectCond: icmp eq SrcVal, 0
- if (!match(SelectCond, m_SpecificICmp(CmpInst::ICMP_EQ, m_Specific(SrcVal),
- m_ZeroInt())))
- continue;
-
- // SelectTrue: the zero-input result
- const APInt *ZeroResult;
- if (!match(SelectTrue, m_APInt(ZeroResult)))
- continue;
-
- // SelectFalse: add/or_disjoint(cttz(trunc(lshr(SrcVal, N/2))), N/2)
- Value *CttzHiCall;
- if (!match(SelectFalse,
- m_c_Add(m_Value(CttzHiCall), m_SpecificInt(HalfWidth))) &&
- !match(SelectFalse,
- m_c_DisjointOr(m_Value(CttzHiCall), m_SpecificInt(HalfWidth))))
- continue;
-
- // CttzHiCall: cttz(trunc(lshr(SrcVal, N/2)), _)
- Value *HiTrunc;
- if (!match(CttzHiCall,
- m_Intrinsic<Intrinsic::cttz>(m_Value(HiTrunc), m_Value())))
- continue;
-
- if (!match(HiTrunc,
- m_Trunc(m_LShr(m_Specific(SrcVal), m_SpecificInt(HalfWidth)))))
- continue;
-
- // Both HiBB and LoBB share a single predecessor
- // that branches on whether the low half is zero.
- BasicBlock *GuardBB = LoBB->getSinglePredecessor();
- if (!GuardBB || GuardBB != HiBB->getSinglePredecessor())
- continue;
-
- Instruction *GuardTerm = GuardBB->getTerminator();
- // Guard: br (icmp eq (trunc SrcVal to i(N/2)), 0), HiBB, LoBB
- if (!match(GuardTerm,
- m_Br(m_SpecificICmp(CmpInst::ICMP_EQ,
- m_Trunc(m_Specific(SrcVal)), m_ZeroInt()),
- m_SpecificBB(HiBB), m_SpecificBB(LoBB))))
- continue;
-
- // Match successful.
- BasicBlock *PhiBB = Phi.getParent();
- IRBuilder<> Builder(PhiBB, PhiBB->getFirstInsertionPt());
-
- Value *CttzWide = Builder.CreateIntrinsic(
- Intrinsic::cttz, {SrcVal->getType()}, {SrcVal, Builder.getTrue()});
- Value *Trunc = Builder.CreateTrunc(CttzWide, HalfTy);
- Value *IsZero =
- Builder.CreateICmpEQ(SrcVal, ConstantInt::get(SrcVal->getType(), 0));
- Value *Result = Builder.CreateSelect(
- IsZero, ConstantInt::get(HalfTy, *ZeroResult), Trunc);
-
- Phi.replaceAllUsesWith(Result);
- ++NumSplitCTTZFolded;
- return true;
- }
-
- return false;
-}
-
-/// Try to fold a phi that merges two half-width ctlz operations into a single
-/// full-width ctlz. This is similar to foldSplitWidthCTTZ, but for leading
-/// zeros.
-///
-/// GuardBB:
-/// %shr = lshr iN %val, N/2
-/// %cmp = icmp eq iN %shr, 0
-/// br i1 %cmp, label %LoBB, label %HiBB
-///
-/// LoBB (upper is zero: count in lower half + N/2):
-/// %lo = trunc iN %val to i(N/2)
-/// %ctlz_lo = call i(N/2) @llvm.ctlz.i(N/2)(i(N/2) %lo, i1 true)
-/// %val_is_zero = icmp eq iN %val, 0
-/// %ctlz_lo_plus = add/or_disjoint i(N/2) %ctlz_lo, N/2
-/// %lo_result = select i1 %val_is_zero, i(N/2) ZeroResult, i(N/2)
-/// %ctlz_lo_plus
-/// br label %PhiBB
-///
-/// HiBB (upper is non-zero: count in upper half):
-/// %hi = trunc iN %shr to i(N/2)
-/// %ctlz_hi = call i(N/2) @llvm.ctlz.i(N/2)(i(N/2) %hi, i1 true)
-/// br label %PhiBB
-///
-/// PhiBB:
-/// %result = phi i(N/2) [%lo_result, %LoBB], [%ctlz_hi, %HiBB]
-/// -->
-/// %ctlz_wide = call iN @llvm.ctlz.iN(iN %val, i1 true)
-/// %trunc = trunc iN %ctlz_wide to i(N/2)
-/// %val_is_zero = icmp eq iN %val, 0
-/// %result = select i1 %val_is_zero, i(N/2) ZeroResult, i(N/2) %trunc
-///
-/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/peel2Y
-static bool foldSplitWidthCTLZ(Instruction &I) {
- if (I.getOpcode() != Instruction::PHI || I.getNumOperands() != 2)
- return false;
-
- Type *HalfTy = I.getType();
- if (!HalfTy->isIntegerTy())
- return false;
- unsigned HalfWidth = HalfTy->getIntegerBitWidth();
- unsigned FullWidth = HalfWidth * 2;
-
- PHINode &Phi = cast<PHINode>(I);
-
- // Try both orderings of phi operands.
- for (unsigned HiIdx = 0; HiIdx < 2; ++HiIdx) {
- unsigned LoIdx = 1 - HiIdx;
- Value *LoVal = Phi.getIncomingValue(LoIdx);
- Value *HiVal = Phi.getIncomingValue(HiIdx);
- BasicBlock *LoBB = Phi.getIncomingBlock(LoIdx);
- BasicBlock *HiBB = Phi.getIncomingBlock(HiIdx);
-
- // Match HiVal: ctlz(trunc(lshr(SrcVal, N/2)), _)
- // This is the ctlz on the upper half (when upper is non-zero).
- Value *HiTrunc;
- if (!match(HiVal,
- m_Intrinsic<Intrinsic::ctlz>(m_Value(HiTrunc), m_Value())))
- continue;
-
- // HiTrunc must be trunc(lshr(SrcVal, N/2)).
- Value *SrcVal = nullptr;
- if (!match(HiTrunc,
- m_Trunc(m_LShr(m_Value(SrcVal), m_SpecificInt(HalfWidth)))))
- continue;
- if (!SrcVal->getType()->isIntegerTy(FullWidth))
- continue;
-
- // Match LoVal: select(icmp eq SrcVal, 0, ZeroResult,
- // add/or_disjoint(ctlz(trunc(SrcVal), _), N/2))
- Value *SelectCond, *SelectTrue, *SelectFalse;
- if (!match(LoVal, m_Select(m_Value(SelectCond), m_Value(SelectTrue),
- m_Value(SelectFalse))))
- continue;
-
- // SelectCond: icmp eq SrcVal, 0
- if (!match(SelectCond, m_SpecificICmp(CmpInst::ICMP_EQ, m_Specific(SrcVal),
- m_ZeroInt())))
- continue;
-
- // SelectTrue: the zero-input result.
- const APInt *ZeroResult;
- if (!match(SelectTrue, m_APInt(ZeroResult)))
- continue;
-
- // SelectFalse: add/or_disjoint(ctlz(trunc(SrcVal), _), N/2)
- Value *CtlzLoCall;
- if (!match(SelectFalse,
- m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(HalfWidth))) &&
- !match(SelectFalse,
- m_c_DisjointOr(m_Value(CtlzLoCall), m_SpecificInt(HalfWidth))))
- continue;
-
- // CtlzLoCall: ctlz(trunc(SrcVal), _)
- Value *LoTrunc;
- if (!match(CtlzLoCall,
- m_Intrinsic<Intrinsic::ctlz>(m_Value(LoTrunc), m_Value())))
- continue;
-
- if (!match(LoTrunc, m_Trunc(m_Specific(SrcVal))))
- continue;
-
- // Both LoBB and HiBB share a single predecessor that branches on whether
- // the upper half is zero.
- BasicBlock *GuardBB = LoBB->getSinglePredecessor();
- if (!GuardBB || GuardBB != HiBB->getSinglePredecessor())
- continue;
-
- Instruction *GuardTerm = GuardBB->getTerminator();
- // Guard: br (icmp eq (lshr SrcVal, N/2), 0), LoBB, HiBB
- if (!match(GuardTerm, m_Br(m_SpecificICmp(CmpInst::ICMP_EQ,
- m_LShr(m_Specific(SrcVal),
- m_SpecificInt(HalfWidth)),
- m_ZeroInt()),
- m_SpecificBB(LoBB), m_SpecificBB(HiBB))))
- continue;
-
- // Match successful.
- BasicBlock *PhiBB = Phi.getParent();
- IRBuilder<> Builder(PhiBB, PhiBB->getFirstInsertionPt());
-
- Value *CtlzWide = Builder.CreateIntrinsic(
- Intrinsic::ctlz, {SrcVal->getType()}, {SrcVal, Builder.getTrue()});
- Value *Trunc = Builder.CreateTrunc(CtlzWide, HalfTy);
- Value *IsZero =
- Builder.CreateICmpEQ(SrcVal, ConstantInt::get(SrcVal->getType(), 0));
- Value *Result = Builder.CreateSelect(
- IsZero, ConstantInt::get(HalfTy, *ZeroResult), Trunc);
-
- Phi.replaceAllUsesWith(Result);
- ++NumSplitCTLZFolded;
- return true;
- }
-
- return false;
-}
-
/// Try to fold a select-based split cttz pattern into a single full-width cttz.
///
/// %lo = trunc iN %val to i(N/2)
@@ -360,7 +85,7 @@ static bool foldSplitWidthCTLZ(Instruction &I) {
/// -->
/// %cttz_wide = call iN @llvm.cttz.iN(iN %val, i1 false)
/// %result = trunc iN %cttz_wide to i(N/2)
-/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/2MkVVM
+/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/-s14-s
static bool foldSelectSplitCTTZ(Instruction &I) {
Value *Cond, *TrueVal, *FalseVal;
if (!match(&I, m_Select(m_Value(Cond), m_Value(TrueVal), m_Value(FalseVal))))
@@ -397,23 +122,21 @@ static bool foldSelectSplitCTTZ(Instruction &I) {
// LoResult: cttz(trunc(SrcVal), _), must use same truncated value
Value *LoCttzArg;
- if (!match(LoResult,
- m_Intrinsic<Intrinsic::cttz>(m_Value(LoCttzArg), m_Value())))
+ if (!match(LoResult, m_OneUse(m_Intrinsic<Intrinsic::cttz>(m_Value(LoCttzArg),
+ m_Value()))))
return false;
if (LoCttzArg != LoTrunc)
return false;
// HiResult: add/or_disjoint(cttz(trunc(lshr(SrcVal, N/2)), _), N/2)
Value *CttzHiCall;
- if (!match(HiResult,
- m_c_Add(m_Value(CttzHiCall), m_SpecificInt(HalfWidth))) &&
- !match(HiResult,
- m_c_DisjointOr(m_Value(CttzHiCall), m_SpecificInt(HalfWidth))))
+ if (!match(HiResult, m_OneUse(m_AddLike(m_Value(CttzHiCall),
+ m_SpecificInt(HalfWidth)))))
return false;
Value *HiCttzArg;
- if (!match(CttzHiCall,
- m_Intrinsic<Intrinsic::cttz>(m_Value(HiCttzArg), m_Value())))
+ if (!match(CttzHiCall, m_OneUse(m_Intrinsic<Intrinsic::cttz>(
+ m_Value(HiCttzArg), m_Value()))))
return false;
if (!match(HiCttzArg,
@@ -445,7 +168,7 @@ static bool foldSelectSplitCTTZ(Instruction &I) {
/// %ctlz_wide = call iN @llvm.ctlz.iN(iN %val, i1 false)
/// %result = trunc iN %ctlz_wide to i(N/2)
///
-/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/rBJygf
+/// Alive proof (for i64/i32): https://alive2.llvm.org/ce/z/WfQepH
static bool foldSelectSplitCTLZ(Instruction &I) {
Value *Cond, *TrueVal, *FalseVal;
if (!match(&I, m_Select(m_Value(Cond), m_Value(TrueVal), m_Value(FalseVal))))
@@ -489,8 +212,8 @@ static bool foldSelectSplitCTLZ(Instruction &I) {
// HiResult: ctlz(trunc(lshr(SrcVal, N/2)), _)
Value *HiCtlzArg;
- if (!match(HiResult,
- m_Intrinsic<Intrinsic::ctlz>(m_Value(HiCtlzArg), m_Value())))
+ if (!match(HiResult, m_OneUse(m_Intrinsic<Intrinsic::ctlz>(m_Value(HiCtlzArg),
+ m_Value()))))
return false;
if (!match(HiCtlzArg,
@@ -499,15 +222,13 @@ static bool foldSelectSplitCTLZ(Instruction &I) {
// LoResult: add/or_disjoint(ctlz(trunc(SrcVal), _), N/2)
Value *CtlzLoCall;
- if (!match(LoResult,
- m_c_Add(m_Value(CtlzLoCall), m_SpecificInt(HalfWidth))) &&
- !match(LoResult,
- m_c_DisjointOr(m_Value(CtlzLoCall), m_SpecificInt(HalfWidth))))
+ if (!match(LoResult, m_OneUse(m_AddLike(m_Value(CtlzLoCall),
+ m_SpecificInt(HalfWidth)))))
return false;
Value *LoCtlzArg;
- if (!match(CtlzLoCall,
- m_Intrinsic<Intrinsic::ctlz>(m_Value(LoCtlzArg), m_Value())))
+ if (!match(CtlzLoCall, m_OneUse(m_Intrinsic<Intrinsic::ctlz>(
+ m_Value(LoCtlzArg), m_Value()))))
return false;
if (!match(LoCtlzArg, m_Trunc(m_Specific(SrcVal))))
@@ -2479,8 +2200,6 @@ static bool foldUnusualPatterns(Function &F, DominatorTree &DT,
for (Instruction &I : make_early_inc_range(llvm::reverse(BB))) {
MadeChange |= foldAnyOrAllBitsSet(I);
MadeChange |= foldGuardedFunnelShift(I, DT);
- MadeChange |= foldSplitWidthCTTZ(I);
- MadeChange |= foldSplitWidthCTLZ(I);
MadeChange |= foldSelectSplitCTTZ(I);
MadeChange |= foldSelectSplitCTLZ(I);
MadeChange |= tryToRecognizePopCount(I);
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
index df6ef312d27c1..5e1ae08226068 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
@@ -1,91 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -passes=aggressive-instcombine -S | FileCheck %s
-; Phi-based split i64 ctlz: branch on whether upper 32 bits are zero,
-; ctlz each half separately, merge via phi.
-define i32 @split_ctlz_phi(i64 noundef %val) {
-; CHECK-LABEL: @split_ctlz_phi(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
-; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK: if.then:
-; CHECK-NEXT: br label [[CLEANUP:%.*]]
-; CHECK: if.else:
-; CHECK-NEXT: br label [[CLEANUP]]
-; CHECK: cleanup:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[VAL]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 63, i32 [[TMP1]]
-; CHECK-NEXT: ret i32 [[TMP3]]
-;
-entry:
- %shr = lshr i64 %val, 32
- %cmp = icmp eq i64 %shr, 0
- br i1 %cmp, label %if.then, label %if.else
-
-if.then:
- %conv1 = trunc nuw i64 %val to i32
- %0 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv1, i1 true)
- %1 = icmp eq i64 %val, 0
- %2 = or disjoint i32 %0, 32
- %add = select i1 %1, i32 63, i32 %2
- br label %cleanup
-
-if.else:
- %conv = trunc nuw i64 %shr to i32
- %3 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv, i1 true)
- br label %cleanup
-
-cleanup:
- %retval.0 = phi i32 [ %add, %if.then ], [ %3, %if.else ]
- ret i32 %retval.0
-}
-
-; Same pattern but with add instead of or.
-define i32 @split_ctlz_phi_add(i64 noundef %val) {
-; CHECK-LABEL: @split_ctlz_phi_add(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[SHR]], 0
-; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK: if.then:
-; CHECK-NEXT: br label [[CLEANUP:%.*]]
-; CHECK: if.else:
-; CHECK-NEXT: br label [[CLEANUP]]
-; CHECK: cleanup:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[VAL]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 63, i32 [[TMP1]]
-; CHECK-NEXT: ret i32 [[TMP3]]
-;
-entry:
- %shr = lshr i64 %val, 32
- %cmp = icmp eq i64 %shr, 0
- br i1 %cmp, label %if.then, label %if.else
-
-if.then:
- %conv1 = trunc nuw i64 %val to i32
- %0 = call i32 @llvm.ctlz.i32(i32 %conv1, i1 false)
- %1 = icmp eq i64 %val, 0
- %2 = add i32 %0, 32
- %add = select i1 %1, i32 63, i32 %2
- br label %cleanup
-
-if.else:
- %conv = trunc nuw i64 %shr to i32
- %3 = call i32 @llvm.ctlz.i32(i32 %conv, i1 true)
- br label %cleanup
-
-cleanup:
- %retval.0 = phi i32 [ %add, %if.then ], [ %3, %if.else ]
- ret i32 %retval.0
-}
-
; Select-based split i64 ctlz
-define i32 @split_ctlz_select(i64 noundef %val) {
+define i32 @split_ctlz_select(i64 %val) {
; CHECK-LABEL: @split_ctlz_select(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL:%.*]], i1 false)
@@ -105,7 +22,7 @@ entry:
}
; Select-based with i64 comparison on lshr result (not truncated).
-define i32 @split_ctlz_select_i64cmp(i64 noundef %val) {
+define i32 @split_ctlz_select_i64cmp(i64 %val) {
; CHECK-LABEL: @split_ctlz_select_i64cmp(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL:%.*]], i1 false)
@@ -125,7 +42,7 @@ entry:
}
; Select-based with icmp ne (inverted condition).
-define i32 @split_ctlz_select_ne(i64 noundef %val) {
+define i32 @split_ctlz_select_ne(i64 %val) {
; CHECK-LABEL: @split_ctlz_select_ne(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL:%.*]], i1 false)
@@ -145,7 +62,7 @@ entry:
}
; Select-based with or instead of add.
-define i32 @split_ctlz_select_or(i64 noundef %val) {
+define i32 @split_ctlz_select_or(i64 %val) {
; CHECK-LABEL: @split_ctlz_select_or(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[VAL:%.*]], i1 false)
@@ -164,49 +81,8 @@ entry:
ret i32 %retval
}
-; i32 split into two i16 ctlz (phi-based).
-define i16 @split_ctlz_phi_i32(i32 noundef %val) {
-; CHECK-LABEL: @split_ctlz_phi_i32(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[VAL:%.*]], 16
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[SHR]], 0
-; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK: if.then:
-; CHECK-NEXT: br label [[CLEANUP:%.*]]
-; CHECK: if.else:
-; CHECK-NEXT: br label [[CLEANUP]]
-; CHECK: cleanup:
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[VAL]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i16
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[VAL]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i16 16, i16 [[TMP1]]
-; CHECK-NEXT: ret i16 [[TMP3]]
-;
-entry:
- %shr = lshr i32 %val, 16
- %cmp = icmp eq i32 %shr, 0
- br i1 %cmp, label %if.then, label %if.else
-
-if.then:
- %conv_lo = trunc i32 %val to i16
- %0 = call i16 @llvm.ctlz.i16(i16 %conv_lo, i1 true)
- %1 = icmp eq i32 %val, 0
- %2 = add i16 %0, 16
- %add = select i1 %1, i16 16, i16 %2
- br label %cleanup
-
-if.else:
- %conv_hi = trunc nuw i32 %shr to i16
- %3 = call i16 @llvm.ctlz.i16(i16 %conv_hi, i1 true)
- br label %cleanup
-
-cleanup:
- %retval.0 = phi i16 [ %add, %if.then ], [ %3, %if.else ]
- ret i16 %retval.0
-}
-
-; i32 split into two i16 ctlz (select-based).
-define i16 @split_ctlz_select_i32(i32 noundef %val) {
+; i32 split into two i16 ctlz.
+define i16 @split_ctlz_select_i32(i32 %val) {
; CHECK-LABEL: @split_ctlz_select_i32(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[VAL:%.*]], i1 false)
@@ -225,8 +101,28 @@ entry:
ret i16 %retval
}
-; i128 split into two i64 ctlz (select-based).
-define i64 @split_ctlz_select_i128(i128 noundef %val) {
+; i16 split into two i8 ctlz.
+define i8 @split_ctlz_select_i16(i16 %val) {
+; CHECK-LABEL: @split_ctlz_select_i16(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i16 @llvm.ctlz.i16(i16 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i16 [[TMP0]] to i8
+; CHECK-NEXT: ret i8 [[TMP1]]
+;
+entry:
+ %shr = lshr i16 %val, 8
+ %conv_hi = trunc nuw i16 %shr to i8
+ %cmp = icmp eq i8 %conv_hi, 0
+ %conv_lo = trunc i16 %val to i8
+ %0 = call i8 @llvm.ctlz.i8(i8 %conv_lo, i1 false)
+ %add = add nuw nsw i8 %0, 8
+ %1 = call i8 @llvm.ctlz.i8(i8 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i8 %add, i8 %1
+ ret i8 %retval
+}
+
+; i128 split into two i64 ctlz.
+define i64 @split_ctlz_select_i128(i128 %val) {
; CHECK-LABEL: @split_ctlz_select_i128(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i128 @llvm.ctlz.i128(i128 [[VAL:%.*]], i1 false)
@@ -245,28 +141,29 @@ entry:
ret i64 %retval
}
-; i16 split into two i8 ctlz (select-based).
-define i8 @split_ctlz_select_i16(i16 noundef %val) {
-; CHECK-LABEL: @split_ctlz_select_i16(
+; i256 split into two i128 ctlz.
+; This tests m_APInt matching for shift amounts > 64 bits.
+define i128 @split_ctlz_select_i256(i256 %val) {
+; CHECK-LABEL: @split_ctlz_select_i256(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = call i16 @llvm.ctlz.i16(i16 [[VAL:%.*]], i1 false)
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i16 [[TMP0]] to i8
-; CHECK-NEXT: ret i8 [[TMP1]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i256 @llvm.ctlz.i256(i256 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i256 [[TMP0]] to i128
+; CHECK-NEXT: ret i128 [[TMP1]]
;
entry:
- %shr = lshr i16 %val, 8
- %conv_hi = trunc nuw i16 %shr to i8
- %cmp = icmp eq i8 %conv_hi, 0
- %conv_lo = trunc i16 %val to i8
- %0 = call i8 @llvm.ctlz.i8(i8 %conv_lo, i1 false)
- %add = add nuw nsw i8 %0, 8
- %1 = call i8 @llvm.ctlz.i8(i8 %conv_hi, i1 true)
- %retval = select i1 %cmp, i8 %add, i8 %1
- ret i8 %retval
+ %shr = lshr i256 %val, 128
+ %conv_hi = trunc nuw i256 %shr to i128
+ %cmp = icmp eq i128 %conv_hi, 0
+ %conv_lo = trunc i256 %val to i128
+ %0 = call i128 @llvm.ctlz.i128(i128 %conv_lo, i1 false)
+ %add = add nuw nsw i128 %0, 128
+ %1 = call i128 @llvm.ctlz.i128(i128 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i128 %add, i128 %1
+ ret i128 %retval
}
; Negative test: mismatched source values, should NOT be folded.
-define i32 @split_ctlz_different_sources(i64 noundef %val, i64 noundef %val2) {
+define i32 @split_ctlz_different_sources(i64 %val, i64 %val2) {
; CHECK-LABEL: @split_ctlz_different_sources(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
@@ -292,7 +189,7 @@ entry:
}
; Negative test: wrong shift amount, should NOT be folded.
-define i32 @split_ctlz_wrong_shift(i64 noundef %val) {
+define i32 @split_ctlz_wrong_shift(i64 %val) {
; CHECK-LABEL: @split_ctlz_wrong_shift(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 16
@@ -319,7 +216,7 @@ entry:
; Negative test: plain or (without disjoint flag), should NOT be folded
; in select-based pattern because or(32, 32)=32 != add(32, 32)=64.
-define i32 @split_ctlz_select_plain_or(i64 noundef %val) {
+define i32 @split_ctlz_select_plain_or(i64 %val) {
; CHECK-LABEL: @split_ctlz_select_plain_or(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
@@ -344,7 +241,33 @@ entry:
ret i32 %retval
}
-declare i8 @llvm.ctlz.i8(i8, i1 immarg)
-declare i16 @llvm.ctlz.i16(i16, i1 immarg)
-declare i32 @llvm.ctlz.i32(i32, i1 immarg)
-declare i64 @llvm.ctlz.i64(i64, i1 immarg)
+; Negative test: multi-use of ctlz_hi prevents folding.
+define i32 @split_ctlz_select_multiuse(i64 %val) {
+; CHECK-LABEL: @split_ctlz_select_multiuse(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc i64 [[VAL]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: call void @use_i32(i32 [[TMP1]])
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %shr = lshr i64 %val, 32
+ %conv_hi = trunc nuw i64 %shr to i32
+ %cmp = icmp eq i32 %conv_hi, 0
+ %conv_lo = trunc i64 %val to i32
+ %0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ call void @use_i32(i32 %1)
+ ret i32 %retval
+}
+
+declare void @use_i32(i32)
+
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
index 15879ec05ab8f..4c759674436e7 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
@@ -1,92 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -passes=aggressive-instcombine -S | FileCheck %s
-; Phi-based split i64 cttz: branch on whether low 32 bits are zero,
-; cttz each half separately, merge via phi.
-define i32 @split_cttz_phi(i64 noundef %val) {
-; CHECK-LABEL: @split_cttz_phi(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
-; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK: if.then:
-; CHECK-NEXT: br label [[CLEANUP:%.*]]
-; CHECK: if.else:
-; CHECK-NEXT: br label [[CLEANUP]]
-; CHECK: cleanup:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[VAL]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 32, i32 [[TMP1]]
-; CHECK-NEXT: ret i32 [[TMP3]]
-;
-entry:
- %conv1 = trunc i64 %val to i32
- %cmp = icmp eq i32 %conv1, 0
- br i1 %cmp, label %if.then, label %if.else
-
-if.then:
- %shr = lshr exact i64 %val, 32
- %conv = trunc nuw i64 %shr to i32
- %0 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv, i1 true)
- %1 = icmp eq i64 %val, 0
- %2 = or disjoint i32 %0, 32
- %add = select i1 %1, i32 32, i32 %2
- br label %cleanup
-
-if.else:
- %3 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
- br label %cleanup
-
-cleanup:
- %retval.0 = phi i32 [ %add, %if.then ], [ %3, %if.else ]
- ret i32 %retval.0
-}
-
-; Same pattern but with add instead of or for the +32.
-define i32 @split_cttz_phi_add(i64 noundef %val) {
-; CHECK-LABEL: @split_cttz_phi_add(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
-; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK: if.then:
-; CHECK-NEXT: br label [[CLEANUP:%.*]]
-; CHECK: if.else:
-; CHECK-NEXT: br label [[CLEANUP]]
-; CHECK: cleanup:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[VAL]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 32, i32 [[TMP1]]
-; CHECK-NEXT: ret i32 [[TMP3]]
-;
-entry:
- %conv1 = trunc i64 %val to i32
- %cmp = icmp eq i32 %conv1, 0
- br i1 %cmp, label %if.then, label %if.else
-
-if.then:
- %shr = lshr exact i64 %val, 32
- %conv = trunc nuw i64 %shr to i32
- %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
- %1 = icmp eq i64 %val, 0
- %2 = add i32 %0, 32
- %add = select i1 %1, i32 32, i32 %2
- br label %cleanup
-
-if.else:
- %3 = call i32 @llvm.cttz.i32(i32 %conv1, i1 true)
- br label %cleanup
-
-cleanup:
- %retval.0 = phi i32 [ %add, %if.then ], [ %3, %if.else ]
- ret i32 %retval.0
-}
-
-; Select-based split i64 cttz: both cttz.i32 are speculated,
-; result picked by select.
-define i32 @split_cttz_select(i64 noundef %val) {
+; Select-based split i64 cttz
+define i32 @split_cttz_select(i64 %val) {
; CHECK-LABEL: @split_cttz_select(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL:%.*]], i1 false)
@@ -106,7 +22,7 @@ entry:
}
; Select-based with or instead of add.
-define i32 @split_cttz_select_or(i64 noundef %val) {
+define i32 @split_cttz_select_or(i64 %val) {
; CHECK-LABEL: @split_cttz_select_or(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL:%.*]], i1 false)
@@ -126,7 +42,7 @@ entry:
}
; Select-based with icmp ne (inverted condition).
-define i32 @split_cttz_select_ne(i64 noundef %val) {
+define i32 @split_cttz_select_ne(i64 %val) {
; CHECK-LABEL: @split_cttz_select_ne(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL:%.*]], i1 false)
@@ -146,7 +62,7 @@ entry:
}
; Select-based cttz used inside a larger expression (>> 3)
-define i32 @split_cttz_select_shift(i64 noundef %val) {
+define i32 @split_cttz_select_shift(i64 %val) {
; CHECK-LABEL: @split_cttz_select_shift(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.cttz.i64(i64 [[VAL:%.*]], i1 false)
@@ -167,49 +83,8 @@ entry:
ret i32 %result
}
-; i32 split into two i16 cttz (phi-based).
-define i16 @split_cttz_phi_i32(i32 noundef %val) {
-; CHECK-LABEL: @split_cttz_phi_i32(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[CONV1:%.*]] = trunc i32 [[VAL:%.*]] to i16
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i16 [[CONV1]], 0
-; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK: if.then:
-; CHECK-NEXT: br label [[CLEANUP:%.*]]
-; CHECK: if.else:
-; CHECK-NEXT: br label [[CLEANUP]]
-; CHECK: cleanup:
-; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[VAL]], i1 true)
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[TMP0]] to i16
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[VAL]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i16 16, i16 [[TMP1]]
-; CHECK-NEXT: ret i16 [[TMP3]]
-;
-entry:
- %conv1 = trunc i32 %val to i16
- %cmp = icmp eq i16 %conv1, 0
- br i1 %cmp, label %if.then, label %if.else
-
-if.then:
- %shr = lshr i32 %val, 16
- %conv = trunc nuw i32 %shr to i16
- %0 = call i16 @llvm.cttz.i16(i16 %conv, i1 true)
- %1 = icmp eq i32 %val, 0
- %2 = add i16 %0, 16
- %add = select i1 %1, i16 16, i16 %2
- br label %cleanup
-
-if.else:
- %3 = call i16 @llvm.cttz.i16(i16 %conv1, i1 true)
- br label %cleanup
-
-cleanup:
- %retval.0 = phi i16 [ %add, %if.then ], [ %3, %if.else ]
- ret i16 %retval.0
-}
-
-; i32 split into two i16 cttz (select-based).
-define i16 @split_cttz_select_i32(i32 noundef %val) {
+; i32 split into two i16 cttz.
+define i16 @split_cttz_select_i32(i32 %val) {
; CHECK-LABEL: @split_cttz_select_i32(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[VAL:%.*]], i1 false)
@@ -228,8 +103,8 @@ entry:
ret i16 %retval
}
-; i128 split into two i64 cttz (select-based).
-define i64 @split_cttz_select_i128(i128 noundef %val) {
+; i128 split into two i64 cttz.
+define i64 @split_cttz_select_i128(i128 %val) {
; CHECK-LABEL: @split_cttz_select_i128(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i128 @llvm.cttz.i128(i128 [[VAL:%.*]], i1 false)
@@ -248,8 +123,8 @@ entry:
ret i64 %retval
}
-; i16 split into two i8 cttz (select-based).
-define i8 @split_cttz_select_i16(i16 noundef %val) {
+; i16 split into two i8 cttz.
+define i8 @split_cttz_select_i16(i16 %val) {
; CHECK-LABEL: @split_cttz_select_i16(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call i16 @llvm.cttz.i16(i16 [[VAL:%.*]], i1 false)
@@ -268,9 +143,29 @@ entry:
ret i8 %retval
}
+; i256 split into two i128 cttz.
+; This tests m_APInt matching for shift amounts > 64 bits.
+define i128 @split_cttz_select_i256(i256 %val) {
+; CHECK-LABEL: @split_cttz_select_i256(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i256 @llvm.cttz.i256(i256 [[VAL:%.*]], i1 false)
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i256 [[TMP0]] to i128
+; CHECK-NEXT: ret i128 [[TMP1]]
+;
+entry:
+ %conv1 = trunc i256 %val to i128
+ %cmp = icmp eq i128 %conv1, 0
+ %shr = lshr i256 %val, 128
+ %conv = trunc nuw i256 %shr to i128
+ %0 = call i128 @llvm.cttz.i128(i128 %conv, i1 false)
+ %add = add nuw nsw i128 %0, 128
+ %1 = call i128 @llvm.cttz.i128(i128 %conv1, i1 true)
+ %retval = select i1 %cmp, i128 %add, i128 %1
+ ret i128 %retval
+}
; Negative test: mismatched source values, should NOT be folded.
-define i32 @split_cttz_different_sources(i64 noundef %val, i64 noundef %val2) {
+define i32 @split_cttz_different_sources(i64 %val, i64 %val2) {
; CHECK-LABEL: @split_cttz_different_sources(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
@@ -296,7 +191,7 @@ entry:
}
; Negative test: shift amount is not 32, should NOT be folded.
-define i32 @split_cttz_wrong_shift(i64 noundef %val) {
+define i32 @split_cttz_wrong_shift(i64 %val) {
; CHECK-LABEL: @split_cttz_wrong_shift(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
@@ -323,7 +218,7 @@ entry:
; Negative test: plain or (without disjoint flag), should NOT be folded
; in select-based pattern because or(32, 32)=32 != add(32, 32)=64.
-define i32 @split_cttz_select_plain_or(i64 noundef %val) {
+define i32 @split_cttz_select_plain_or(i64 %val) {
; CHECK-LABEL: @split_cttz_select_plain_or(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
@@ -348,7 +243,32 @@ entry:
ret i32 %retval
}
-declare i8 @llvm.cttz.i8(i8, i1 immarg)
-declare i16 @llvm.cttz.i16(i16, i1 immarg)
-declare i32 @llvm.cttz.i32(i32, i1 immarg)
-declare i64 @llvm.cttz.i64(i64, i1 immarg)
+; Negative test: multi-use of cttz_lo prevents folding.
+define i32 @split_cttz_select_multiuse(i64 %val) {
+; CHECK-LABEL: @split_cttz_select_multiuse(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL]], 32
+; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
+; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
+; CHECK-NEXT: call void @use_i32(i32 [[TMP1]])
+; CHECK-NEXT: ret i32 [[RETVAL]]
+;
+entry:
+ %conv1 = trunc i64 %val to i32
+ %cmp = icmp eq i32 %conv1, 0
+ %shr = lshr i64 %val, 32
+ %conv = trunc nuw i64 %shr to i32
+ %0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
+ %add = add nuw nsw i32 %0, 32
+ %1 = call i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %retval = select i1 %cmp, i32 %add, i32 %1
+ call void @use_i32(i32 %1)
+ ret i32 %retval
+}
+
+declare void @use_i32(i32)
>From f911f4c05101ac9b4912c90be36f58a3bb96018d Mon Sep 17 00:00:00 2001
From: Usha Gupta <usha.gupta at arm.com>
Date: Wed, 22 Apr 2026 10:12:31 +0000
Subject: [PATCH 5/6] Update tests to drop unnecessary attributes
---
.../AggressiveInstCombine/fold-split-ctlz.ll | 98 +++++++++----------
.../AggressiveInstCombine/fold-split-cttz.ll | 86 ++++++++--------
2 files changed, 92 insertions(+), 92 deletions(-)
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
index 5e1ae08226068..73e522f535b65 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-ctlz.ll
@@ -11,12 +11,12 @@ define i32 @split_ctlz_select(i64 %val) {
;
entry:
%shr = lshr i64 %val, 32
- %conv_hi = trunc nuw i64 %shr to i32
+ %conv_hi = trunc i64 %shr to i32
%cmp = icmp eq i32 %conv_hi, 0
- %conv_lo = trunc nuw i64 %val to i32
+ %conv_lo = trunc i64 %val to i32
%0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
- %add = add nuw nsw i32 %0, 32
- %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %add = add i32 %0, 32
+ %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
ret i32 %retval
}
@@ -32,11 +32,11 @@ define i32 @split_ctlz_select_i64cmp(i64 %val) {
entry:
%shr = lshr i64 %val, 32
%cmp = icmp eq i64 %shr, 0
- %conv_hi = trunc nuw i64 %shr to i32
- %conv_lo = trunc nuw i64 %val to i32
+ %conv_hi = trunc i64 %shr to i32
+ %conv_lo = trunc i64 %val to i32
%0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
- %add = add nuw nsw i32 %0, 32
- %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %add = add i32 %0, 32
+ %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
ret i32 %retval
}
@@ -51,12 +51,12 @@ define i32 @split_ctlz_select_ne(i64 %val) {
;
entry:
%shr = lshr i64 %val, 32
- %conv_hi = trunc nuw i64 %shr to i32
+ %conv_hi = trunc i64 %shr to i32
%cmp = icmp ne i32 %conv_hi, 0
- %conv_lo = trunc nuw i64 %val to i32
+ %conv_lo = trunc i64 %val to i32
%0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
- %add = add nuw nsw i32 %0, 32
- %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %add = add i32 %0, 32
+ %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 false)
%retval = select i1 %cmp, i32 %1, i32 %add
ret i32 %retval
}
@@ -71,12 +71,12 @@ define i32 @split_ctlz_select_or(i64 %val) {
;
entry:
%shr = lshr i64 %val, 32
- %conv_hi = trunc nuw i64 %shr to i32
+ %conv_hi = trunc i64 %shr to i32
%cmp = icmp eq i32 %conv_hi, 0
- %conv_lo = trunc nuw i64 %val to i32
+ %conv_lo = trunc i64 %val to i32
%0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
%add = or disjoint i32 %0, 32
- %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
ret i32 %retval
}
@@ -91,12 +91,12 @@ define i16 @split_ctlz_select_i32(i32 %val) {
;
entry:
%shr = lshr i32 %val, 16
- %conv_hi = trunc nuw i32 %shr to i16
+ %conv_hi = trunc i32 %shr to i16
%cmp = icmp eq i16 %conv_hi, 0
%conv_lo = trunc i32 %val to i16
%0 = call i16 @llvm.ctlz.i16(i16 %conv_lo, i1 false)
- %add = add nuw nsw i16 %0, 16
- %1 = call i16 @llvm.ctlz.i16(i16 %conv_hi, i1 true)
+ %add = add i16 %0, 16
+ %1 = call i16 @llvm.ctlz.i16(i16 %conv_hi, i1 false)
%retval = select i1 %cmp, i16 %add, i16 %1
ret i16 %retval
}
@@ -111,12 +111,12 @@ define i8 @split_ctlz_select_i16(i16 %val) {
;
entry:
%shr = lshr i16 %val, 8
- %conv_hi = trunc nuw i16 %shr to i8
+ %conv_hi = trunc i16 %shr to i8
%cmp = icmp eq i8 %conv_hi, 0
%conv_lo = trunc i16 %val to i8
%0 = call i8 @llvm.ctlz.i8(i8 %conv_lo, i1 false)
- %add = add nuw nsw i8 %0, 8
- %1 = call i8 @llvm.ctlz.i8(i8 %conv_hi, i1 true)
+ %add = add i8 %0, 8
+ %1 = call i8 @llvm.ctlz.i8(i8 %conv_hi, i1 false)
%retval = select i1 %cmp, i8 %add, i8 %1
ret i8 %retval
}
@@ -131,12 +131,12 @@ define i64 @split_ctlz_select_i128(i128 %val) {
;
entry:
%shr = lshr i128 %val, 64
- %conv_hi = trunc nuw i128 %shr to i64
+ %conv_hi = trunc i128 %shr to i64
%cmp = icmp eq i64 %conv_hi, 0
%conv_lo = trunc i128 %val to i64
%0 = call i64 @llvm.ctlz.i64(i64 %conv_lo, i1 false)
- %add = add nuw nsw i64 %0, 64
- %1 = call i64 @llvm.ctlz.i64(i64 %conv_hi, i1 true)
+ %add = add i64 %0, 64
+ %1 = call i64 @llvm.ctlz.i64(i64 %conv_hi, i1 false)
%retval = select i1 %cmp, i64 %add, i64 %1
ret i64 %retval
}
@@ -152,12 +152,12 @@ define i128 @split_ctlz_select_i256(i256 %val) {
;
entry:
%shr = lshr i256 %val, 128
- %conv_hi = trunc nuw i256 %shr to i128
+ %conv_hi = trunc i256 %shr to i128
%cmp = icmp eq i128 %conv_hi, 0
%conv_lo = trunc i256 %val to i128
%0 = call i128 @llvm.ctlz.i128(i128 %conv_lo, i1 false)
- %add = add nuw nsw i128 %0, 128
- %1 = call i128 @llvm.ctlz.i128(i128 %conv_hi, i1 true)
+ %add = add i128 %0, 128
+ %1 = call i128 @llvm.ctlz.i128(i128 %conv_hi, i1 false)
%retval = select i1 %cmp, i128 %add, i128 %1
ret i128 %retval
}
@@ -167,23 +167,23 @@ define i32 @split_ctlz_different_sources(i64 %val, i64 %val2) {
; CHECK-LABEL: @split_ctlz_different_sources(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
-; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc i64 [[SHR]] to i32
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
-; CHECK-NEXT: [[CONV_LO:%.*]] = trunc nuw i64 [[VAL2:%.*]] to i32
+; CHECK-NEXT: [[CONV_LO:%.*]] = trunc i64 [[VAL2:%.*]] to i32
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 false)
; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[RETVAL]]
;
entry:
%shr = lshr i64 %val, 32
- %conv_hi = trunc nuw i64 %shr to i32
+ %conv_hi = trunc i64 %shr to i32
%cmp = icmp eq i32 %conv_hi, 0
- %conv_lo = trunc nuw i64 %val2 to i32
+ %conv_lo = trunc i64 %val2 to i32
%0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
- %add = add nuw nsw i32 %0, 32
- %1 = call range(i32 0, 33) i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %add = add i32 %0, 32
+ %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
ret i32 %retval
}
@@ -197,8 +197,8 @@ define i32 @split_ctlz_wrong_shift(i64 %val) {
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
; CHECK-NEXT: [[CONV_LO:%.*]] = trunc i64 [[VAL]] to i32
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 false)
; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[RETVAL]]
;
@@ -208,8 +208,8 @@ entry:
%cmp = icmp eq i32 %conv_hi, 0
%conv_lo = trunc i64 %val to i32
%0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
- %add = add nuw nsw i32 %0, 32
- %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %add = add i32 %0, 32
+ %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
ret i32 %retval
}
@@ -220,23 +220,23 @@ define i32 @split_ctlz_select_plain_or(i64 %val) {
; CHECK-LABEL: @split_ctlz_select_plain_or(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
-; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc i64 [[SHR]] to i32
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
; CHECK-NEXT: [[CONV_LO:%.*]] = trunc i64 [[VAL]] to i32
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 false)
; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[RETVAL]]
;
entry:
%shr = lshr i64 %val, 32
- %conv_hi = trunc nuw i64 %shr to i32
+ %conv_hi = trunc i64 %shr to i32
%cmp = icmp eq i32 %conv_hi, 0
%conv_lo = trunc i64 %val to i32
%0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
%add = or i32 %0, 32
- %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
ret i32 %retval
}
@@ -246,24 +246,24 @@ define i32 @split_ctlz_select_multiuse(i64 %val) {
; CHECK-LABEL: @split_ctlz_select_multiuse(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL:%.*]], 32
-; CHECK-NEXT: [[CONV_HI:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CONV_HI:%.*]] = trunc i64 [[SHR]] to i32
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV_HI]], 0
; CHECK-NEXT: [[CONV_LO:%.*]] = trunc i64 [[VAL]] to i32
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_LO]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 true)
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[CONV_HI]], i1 false)
; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
; CHECK-NEXT: call void @use_i32(i32 [[TMP1]])
; CHECK-NEXT: ret i32 [[RETVAL]]
;
entry:
%shr = lshr i64 %val, 32
- %conv_hi = trunc nuw i64 %shr to i32
+ %conv_hi = trunc i64 %shr to i32
%cmp = icmp eq i32 %conv_hi, 0
%conv_lo = trunc i64 %val to i32
%0 = call i32 @llvm.ctlz.i32(i32 %conv_lo, i1 false)
- %add = add nuw nsw i32 %0, 32
- %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 true)
+ %add = add i32 %0, 32
+ %1 = call i32 @llvm.ctlz.i32(i32 %conv_hi, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
call void @use_i32(i32 %1)
ret i32 %retval
diff --git a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
index 4c759674436e7..b46b721acc862 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/fold-split-cttz.ll
@@ -13,10 +13,10 @@ entry:
%conv1 = trunc i64 %val to i32
%cmp = icmp eq i32 %conv1, 0
%shr = lshr i64 %val, 32
- %conv = trunc nuw i64 %shr to i32
+ %conv = trunc i64 %shr to i32
%0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
- %add = add nuw nsw i32 %0, 32
- %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %add = add i32 %0, 32
+ %1 = call i32 @llvm.cttz.i32(i32 %conv1, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
ret i32 %retval
}
@@ -33,10 +33,10 @@ entry:
%conv1 = trunc i64 %val to i32
%cmp = icmp eq i32 %conv1, 0
%shr = lshr i64 %val, 32
- %conv = trunc nuw i64 %shr to i32
+ %conv = trunc i64 %shr to i32
%0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
%add = or disjoint i32 %0, 32
- %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %1 = call i32 @llvm.cttz.i32(i32 %conv1, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
ret i32 %retval
}
@@ -53,10 +53,10 @@ entry:
%conv1 = trunc i64 %val to i32
%cmp = icmp ne i32 %conv1, 0
%shr = lshr i64 %val, 32
- %conv = trunc nuw i64 %shr to i32
+ %conv = trunc i64 %shr to i32
%0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
- %add = add nuw nsw i32 %0, 32
- %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %add = add i32 %0, 32
+ %1 = call i32 @llvm.cttz.i32(i32 %conv1, i1 false)
%retval = select i1 %cmp, i32 %1, i32 %add
ret i32 %retval
}
@@ -74,10 +74,10 @@ entry:
%conv1 = trunc i64 %val to i32
%cmp = icmp eq i32 %conv1, 0
%shr = lshr i64 %val, 32
- %conv = trunc nuw i64 %shr to i32
+ %conv = trunc i64 %shr to i32
%0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
- %add = add nuw nsw i32 %0, 32
- %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %add = add i32 %0, 32
+ %1 = call i32 @llvm.cttz.i32(i32 %conv1, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
%result = lshr i32 %retval, 3
ret i32 %result
@@ -95,10 +95,10 @@ entry:
%conv1 = trunc i32 %val to i16
%cmp = icmp eq i16 %conv1, 0
%shr = lshr i32 %val, 16
- %conv = trunc nuw i32 %shr to i16
+ %conv = trunc i32 %shr to i16
%0 = call i16 @llvm.cttz.i16(i16 %conv, i1 false)
- %add = add nuw nsw i16 %0, 16
- %1 = call i16 @llvm.cttz.i16(i16 %conv1, i1 true)
+ %add = add i16 %0, 16
+ %1 = call i16 @llvm.cttz.i16(i16 %conv1, i1 false)
%retval = select i1 %cmp, i16 %add, i16 %1
ret i16 %retval
}
@@ -115,10 +115,10 @@ entry:
%conv1 = trunc i128 %val to i64
%cmp = icmp eq i64 %conv1, 0
%shr = lshr i128 %val, 64
- %conv = trunc nuw i128 %shr to i64
+ %conv = trunc i128 %shr to i64
%0 = call i64 @llvm.cttz.i64(i64 %conv, i1 false)
- %add = add nuw nsw i64 %0, 64
- %1 = call i64 @llvm.cttz.i64(i64 %conv1, i1 true)
+ %add = add i64 %0, 64
+ %1 = call i64 @llvm.cttz.i64(i64 %conv1, i1 false)
%retval = select i1 %cmp, i64 %add, i64 %1
ret i64 %retval
}
@@ -135,10 +135,10 @@ entry:
%conv1 = trunc i16 %val to i8
%cmp = icmp eq i8 %conv1, 0
%shr = lshr i16 %val, 8
- %conv = trunc nuw i16 %shr to i8
+ %conv = trunc i16 %shr to i8
%0 = call i8 @llvm.cttz.i8(i8 %conv, i1 false)
- %add = add nuw nsw i8 %0, 8
- %1 = call i8 @llvm.cttz.i8(i8 %conv1, i1 true)
+ %add = add i8 %0, 8
+ %1 = call i8 @llvm.cttz.i8(i8 %conv1, i1 false)
%retval = select i1 %cmp, i8 %add, i8 %1
ret i8 %retval
}
@@ -156,10 +156,10 @@ entry:
%conv1 = trunc i256 %val to i128
%cmp = icmp eq i128 %conv1, 0
%shr = lshr i256 %val, 128
- %conv = trunc nuw i256 %shr to i128
+ %conv = trunc i256 %shr to i128
%0 = call i128 @llvm.cttz.i128(i128 %conv, i1 false)
- %add = add nuw nsw i128 %0, 128
- %1 = call i128 @llvm.cttz.i128(i128 %conv1, i1 true)
+ %add = add i128 %0, 128
+ %1 = call i128 @llvm.cttz.i128(i128 %conv1, i1 false)
%retval = select i1 %cmp, i128 %add, i128 %1
ret i128 %retval
}
@@ -171,10 +171,10 @@ define i32 @split_cttz_different_sources(i64 %val, i64 %val2) {
; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL2:%.*]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CONV:%.*]] = trunc i64 [[SHR]] to i32
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV1]], i1 false)
; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[RETVAL]]
;
@@ -182,10 +182,10 @@ entry:
%conv1 = trunc i64 %val to i32
%cmp = icmp eq i32 %conv1, 0
%shr = lshr i64 %val2, 32
- %conv = trunc nuw i64 %shr to i32
+ %conv = trunc i64 %shr to i32
%0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
- %add = add nuw nsw i32 %0, 32
- %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %add = add i32 %0, 32
+ %1 = call i32 @llvm.cttz.i32(i32 %conv1, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
ret i32 %retval
}
@@ -199,8 +199,8 @@ define i32 @split_cttz_wrong_shift(i64 %val) {
; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL]], 16
; CHECK-NEXT: [[CONV:%.*]] = trunc i64 [[SHR]] to i32
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV1]], i1 false)
; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[RETVAL]]
;
@@ -210,8 +210,8 @@ entry:
%shr = lshr i64 %val, 16
%conv = trunc i64 %shr to i32
%0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
- %add = add nuw nsw i32 %0, 32
- %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %add = add i32 %0, 32
+ %1 = call i32 @llvm.cttz.i32(i32 %conv1, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
ret i32 %retval
}
@@ -224,10 +224,10 @@ define i32 @split_cttz_select_plain_or(i64 %val) {
; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CONV:%.*]] = trunc i64 [[SHR]] to i32
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP1:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV1]], i1 false)
; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
; CHECK-NEXT: ret i32 [[RETVAL]]
;
@@ -235,10 +235,10 @@ entry:
%conv1 = trunc i64 %val to i32
%cmp = icmp eq i32 %conv1, 0
%shr = lshr i64 %val, 32
- %conv = trunc nuw i64 %shr to i32
+ %conv = trunc i64 %shr to i32
%0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
%add = or i32 %0, 32
- %1 = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %1 = call i32 @llvm.cttz.i32(i32 %conv1, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
ret i32 %retval
}
@@ -250,10 +250,10 @@ define i32 @split_cttz_select_multiuse(i64 %val) {
; CHECK-NEXT: [[CONV1:%.*]] = trunc i64 [[VAL:%.*]] to i32
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[VAL]], 32
-; CHECK-NEXT: [[CONV:%.*]] = trunc nuw i64 [[SHR]] to i32
+; CHECK-NEXT: [[CONV:%.*]] = trunc i64 [[SHR]] to i32
; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV]], i1 false)
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[TMP0]], 32
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV1]], i1 true)
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[TMP0]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.cttz.i32(i32 [[CONV1]], i1 false)
; CHECK-NEXT: [[RETVAL:%.*]] = select i1 [[CMP]], i32 [[ADD]], i32 [[TMP1]]
; CHECK-NEXT: call void @use_i32(i32 [[TMP1]])
; CHECK-NEXT: ret i32 [[RETVAL]]
@@ -262,10 +262,10 @@ entry:
%conv1 = trunc i64 %val to i32
%cmp = icmp eq i32 %conv1, 0
%shr = lshr i64 %val, 32
- %conv = trunc nuw i64 %shr to i32
+ %conv = trunc i64 %shr to i32
%0 = call i32 @llvm.cttz.i32(i32 %conv, i1 false)
- %add = add nuw nsw i32 %0, 32
- %1 = call i32 @llvm.cttz.i32(i32 %conv1, i1 true)
+ %add = add i32 %0, 32
+ %1 = call i32 @llvm.cttz.i32(i32 %conv1, i1 false)
%retval = select i1 %cmp, i32 %add, i32 %1
call void @use_i32(i32 %1)
ret i32 %retval
>From b380ea2761606d557e292784b8786d7085e4f3c0 Mon Sep 17 00:00:00 2001
From: Usha Gupta <usha.gupta at arm.com>
Date: Thu, 23 Apr 2026 11:04:12 +0000
Subject: [PATCH 6/6] Prevent this transform for very small types
---
.../AggressiveInstCombine.cpp | 21 ++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index 077168878d501..51a994b7ab9f3 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -95,6 +95,13 @@ static bool foldSelectSplitCTTZ(Instruction &I) {
if (!HalfTy->isIntegerTy())
return false;
unsigned HalfWidth = HalfTy->getIntegerBitWidth();
+
+ // Bail out on very small types (i1, i2): the full-width cttz can return
+ // values not representable in the half type (e.g., cttz.i4 can return 4,
+ // which doesn't fit in i2).
+ if (HalfWidth <= 2)
+ return false;
+
unsigned FullWidth = HalfWidth * 2;
// select (icmp eq (trunc SrcVal to i(N/2)), 0), HiResult, LoResult
@@ -121,11 +128,8 @@ static bool foldSelectSplitCTTZ(Instruction &I) {
return false;
// LoResult: cttz(trunc(SrcVal), _), must use same truncated value
- Value *LoCttzArg;
- if (!match(LoResult, m_OneUse(m_Intrinsic<Intrinsic::cttz>(m_Value(LoCttzArg),
- m_Value()))))
- return false;
- if (LoCttzArg != LoTrunc)
+ if (!match(LoResult, m_OneUse(m_Intrinsic<Intrinsic::cttz>(
+ m_Specific(LoTrunc), m_Value()))))
return false;
// HiResult: add/or_disjoint(cttz(trunc(lshr(SrcVal, N/2)), _), N/2)
@@ -178,6 +182,13 @@ static bool foldSelectSplitCTLZ(Instruction &I) {
if (!HalfTy->isIntegerTy())
return false;
unsigned HalfWidth = HalfTy->getIntegerBitWidth();
+
+ // Bail out on very small types (i1, i2): the full-width ctlz can return
+ // values not representable in the half type (e.g., ctlz.i4 can return 4,
+ // which doesn't fit in i2).
+ if (HalfWidth <= 2)
+ return false;
+
unsigned FullWidth = HalfWidth * 2;
// select (icmp eq HiPart, 0), LoResult, HiResult
More information about the llvm-commits
mailing list