[llvm] [SeparateConstOffsetFromGEP] Fix sext/add overflow checks (PR #228562)
Alex MacLean via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 5 12:46:18 PDT 2026
https://github.com/AlexMaclean updated https://github.com/llvm/llvm-project/pull/228562
>From c98993da88cbc9c16e35ba9fcf6218fed45edb3f Mon Sep 17 00:00:00 2001
From: Alex Maclean <amaclean at nvidia.com>
Date: Fri, 2 Oct 2026 12:30:33 -0700
Subject: [PATCH 1/3] [SeparateConstOffsetFromGEP] Precommit tests for sext/add
miscompiles
---
.../NVPTX/sext-add-inbounds.ll | 110 ++++++++++++
.../NVPTX/sext-add-nonnegative.ll | 166 ++++++++++++++++++
2 files changed, 276 insertions(+)
create mode 100644 llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-inbounds.ll
create mode 100644 llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-nonnegative.ll
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-inbounds.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-inbounds.ll
new file mode 100644
index 0000000000000..6af5f7c977557
--- /dev/null
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-inbounds.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=nvptx64-nvidia-cuda -passes=separate-const-offset-from-gep -verify-each -reassociate-geps-verify-no-dead-code < %s | FileCheck %s --check-prefixes=CHECK,SPLIT
+; RUN: opt -S -mtriple=nvptx64-nvidia-cuda -passes='separate-const-offset-from-gep<lower-gep>' -verify-each -reassociate-geps-verify-no-dead-code < %s | FileCheck %s --check-prefixes=CHECK,LOWER
+
+ at array = global [256 x i8] zeroinitializer
+
+; An earlier index can move the pointer away from the object base. For x = 127,
+; the sum wraps to -128 and the original GEP returns @array. Distributing sext
+; would instead return @array + 256. The first index is extracted before the
+; pass rewrites the second index, so it must not use the rewritten zero index
+; to infer that the second index is nonnegative.
+define ptr @preceding_index(i8 %x) {
+; SPLIT-LABEL: define ptr @preceding_index(
+; SPLIT-SAME: i8 [[X:%.*]]) {
+; SPLIT-NEXT: [[INDEX:%.*]] = sext i8 [[X]] to i64
+; SPLIT-NEXT: [[TMP1:%.*]] = getelementptr [128 x i8], ptr @array, i64 0, i64 [[INDEX]]
+; SPLIT-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[TMP1]], i64 129
+; SPLIT-NEXT: ret ptr [[P1]]
+;
+; LOWER-LABEL: define ptr @preceding_index(
+; LOWER-SAME: i8 [[X:%.*]]) {
+; LOWER-NEXT: [[INDEX:%.*]] = sext i8 [[X]] to i64
+; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr @array, i64 [[INDEX]]
+; LOWER-NEXT: [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 129
+; LOWER-NEXT: ret ptr [[UGLYGEP1]]
+;
+ %sum = add i8 %x, 1
+ %index = sext i8 %sum to i64
+ %p = getelementptr inbounds [128 x i8], ptr @array, i64 1, i64 %index
+ ret ptr %p
+}
+
+; The second index has stride 1, not 256. For x = -128, signed underflow gives
+; a valid offset of 127 into this object. Distributing sext would give -129.
+define ptr @negative_offset(i8 %x) {
+; SPLIT-LABEL: define ptr @negative_offset(
+; SPLIT-SAME: i8 [[X:%.*]]) {
+; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[X]] to i64
+; SPLIT-NEXT: [[TMP2:%.*]] = getelementptr [256 x i8], ptr @array, i64 0, i64 [[TMP1]]
+; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -1
+; SPLIT-NEXT: ret ptr [[P2]]
+;
+; LOWER-LABEL: define ptr @negative_offset(
+; LOWER-SAME: i8 [[X:%.*]]) {
+; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[X]] to i64
+; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr @array, i64 [[TMP1]]
+; LOWER-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 -1
+; LOWER-NEXT: ret ptr [[UGLYGEP2]]
+;
+ %sum = add i8 %x, -1
+ %index = sext i8 %sum to i64
+ %p = getelementptr inbounds [256 x i8], ptr @array, i64 0, i64 %index
+ ret ptr %p
+}
+
+; The pointer starts 128 bytes into the object, so an index of -128 is valid.
+; Again, using the size of the entire array as the stride would incorrectly
+; rule out signed overflow of the add.
+define ptr @interior_base(i8 %x) {
+; SPLIT-LABEL: define ptr @interior_base(
+; SPLIT-SAME: i8 [[X:%.*]]) {
+; SPLIT-NEXT: [[INDEX:%.*]] = sext i8 [[X]] to i64
+; SPLIT-NEXT: [[TMP1:%.*]] = getelementptr [256 x i8], ptr @array, i64 0, i64 [[INDEX]]
+; SPLIT-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[TMP1]], i64 129
+; SPLIT-NEXT: ret ptr [[P1]]
+;
+; LOWER-LABEL: define ptr @interior_base(
+; LOWER-SAME: i8 [[X:%.*]]) {
+; LOWER-NEXT: [[INDEX:%.*]] = sext i8 [[X]] to i64
+; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr @array, i64 [[INDEX]]
+; LOWER-NEXT: [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 129
+; LOWER-NEXT: ret ptr [[UGLYGEP1]]
+;
+ %base = getelementptr i8, ptr @array, i64 128
+ %sum = add i8 %x, 1
+ %index = sext i8 %sum to i64
+ %p = getelementptr inbounds [256 x i8], ptr %base, i64 0, i64 %index
+ ret ptr %p
+}
+
+; Identical index values can have different strides. The first index's stride
+; rules out signed underflow, but the second index's stride does not. Finding
+; and extracting offsets must use the same stride even after the first index
+; has been rewritten.
+define ptr @repeated_index(i8 %x) {
+; SPLIT-LABEL: define ptr @repeated_index(
+; SPLIT-SAME: i8 [[X:%.*]]) {
+; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[X]] to i64
+; SPLIT-NEXT: [[TMP2:%.*]] = sext i8 [[X]] to i64
+; SPLIT-NEXT: [[TMP3:%.*]] = getelementptr [256 x i8], ptr @array, i64 [[TMP2]], i64 [[TMP1]]
+; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP3]], i64 -257
+; SPLIT-NEXT: ret ptr [[P2]]
+;
+; LOWER-LABEL: define ptr @repeated_index(
+; LOWER-SAME: i8 [[X:%.*]]) {
+; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[X]] to i64
+; LOWER-NEXT: [[TMP3:%.*]] = sext i8 [[X]] to i64
+; LOWER-NEXT: [[TMP2:%.*]] = shl i64 [[TMP3]], 8
+; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr @array, i64 [[TMP2]]
+; LOWER-NEXT: [[UGLYGEP4:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 [[TMP1]]
+; LOWER-NEXT: [[UGLYGEP3:%.*]] = getelementptr i8, ptr [[UGLYGEP4]], i64 -257
+; LOWER-NEXT: ret ptr [[UGLYGEP3]]
+;
+ %sum = add i8 %x, -1
+ %index = sext i8 %sum to i64
+ %p = getelementptr inbounds [256 x i8], ptr @array, i64 %index, i64 %index
+ ret ptr %p
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-nonnegative.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-nonnegative.ll
new file mode 100644
index 0000000000000..efb27da9ff9bb
--- /dev/null
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-nonnegative.ll
@@ -0,0 +1,166 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=nvptx64-nvidia-cuda -passes=separate-const-offset-from-gep -verify-each -reassociate-geps-verify-no-dead-code < %s | FileCheck %s --check-prefixes=CHECK,SPLIT
+; RUN: opt -S -mtriple=nvptx64-nvidia-cuda -passes='separate-const-offset-from-gep<lower-gep>' -verify-each -reassociate-geps-verify-no-dead-code < %s | FileCheck %s --check-prefixes=CHECK,LOWER
+
+; A nonnegative sum of two negative operands may have signed overflow.
+; For x = 0, the original index is 0, but distributing sext would give -256.
+define ptr @negative_operands(ptr %base, i8 %x) {
+; SPLIT-LABEL: define ptr @negative_operands(
+; SPLIT-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; SPLIT-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
+; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i64
+; SPLIT-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
+; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -128
+; SPLIT-NEXT: ret ptr [[P2]]
+;
+; LOWER-LABEL: define ptr @negative_operands(
+; LOWER-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; LOWER-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
+; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i64
+; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
+; LOWER-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 -128
+; LOWER-NEXT: ret ptr [[UGLYGEP2]]
+;
+ %negative = or i8 %x, -128
+ %sum = add i8 %negative, -128
+ %index = sext i8 %sum to i64
+ %p = getelementptr i8, ptr %base, i64 %index
+ ret ptr %p
+}
+
+define ptr @negative_operands_commuted(ptr %base, i8 %x) {
+; SPLIT-LABEL: define ptr @negative_operands_commuted(
+; SPLIT-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; SPLIT-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
+; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i64
+; SPLIT-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
+; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -128
+; SPLIT-NEXT: ret ptr [[P2]]
+;
+; LOWER-LABEL: define ptr @negative_operands_commuted(
+; LOWER-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; LOWER-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
+; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i64
+; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
+; LOWER-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 -128
+; LOWER-NEXT: ret ptr [[UGLYGEP2]]
+;
+ %negative = or i8 %x, -128
+ %sum = add i8 -128, %negative
+ %index = sext i8 %sum to i64
+ %p = getelementptr i8, ptr %base, i64 %index
+ ret ptr %p
+}
+
+; A nonnegative sum with a nonnegative operand cannot have signed overflow.
+; The positive operand is in [64, 127], so the sum is in [0, 63].
+define ptr @nonnegative_lhs(ptr %base, i8 %x) {
+; SPLIT-LABEL: define ptr @nonnegative_lhs(
+; SPLIT-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; SPLIT-NEXT: [[MASKED:%.*]] = and i8 [[X]], 63
+; SPLIT-NEXT: [[POSITIVE:%.*]] = or i8 [[MASKED]], 64
+; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[POSITIVE]] to i64
+; SPLIT-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
+; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -64
+; SPLIT-NEXT: ret ptr [[P2]]
+;
+; LOWER-LABEL: define ptr @nonnegative_lhs(
+; LOWER-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; LOWER-NEXT: [[MASKED:%.*]] = and i8 [[X]], 63
+; LOWER-NEXT: [[POSITIVE:%.*]] = or i8 [[MASKED]], 64
+; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[POSITIVE]] to i64
+; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
+; LOWER-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 -64
+; LOWER-NEXT: ret ptr [[UGLYGEP2]]
+;
+ %masked = and i8 %x, 63
+ %positive = or i8 %masked, 64
+ %sum = add i8 %positive, -64
+ %index = sext i8 %sum to i64
+ %p = getelementptr i8, ptr %base, i64 %index
+ ret ptr %p
+}
+
+define ptr @nonnegative_rhs(ptr %base, i8 %x) {
+; SPLIT-LABEL: define ptr @nonnegative_rhs(
+; SPLIT-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; SPLIT-NEXT: [[MASKED:%.*]] = and i8 [[X]], 63
+; SPLIT-NEXT: [[POSITIVE:%.*]] = or i8 [[MASKED]], 64
+; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[POSITIVE]] to i64
+; SPLIT-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
+; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -64
+; SPLIT-NEXT: ret ptr [[P2]]
+;
+; LOWER-LABEL: define ptr @nonnegative_rhs(
+; LOWER-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; LOWER-NEXT: [[MASKED:%.*]] = and i8 [[X]], 63
+; LOWER-NEXT: [[POSITIVE:%.*]] = or i8 [[MASKED]], 64
+; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[POSITIVE]] to i64
+; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
+; LOWER-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 -64
+; LOWER-NEXT: ret ptr [[UGLYGEP2]]
+;
+ %masked = and i8 %x, 63
+ %positive = or i8 %masked, 64
+ %sum = add i8 -64, %positive
+ %index = sext i8 %sum to i64
+ %p = getelementptr i8, ptr %base, i64 %index
+ ret ptr %p
+}
+
+; Recursing through sexts must still check the sign of an add operand.
+define ptr @negative_operands_sext_chain(ptr %base, i8 %x) {
+; SPLIT-LABEL: define ptr @negative_operands_sext_chain(
+; SPLIT-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; SPLIT-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
+; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i32
+; SPLIT-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
+; SPLIT-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP2]]
+; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP3]], i64 -128
+; SPLIT-NEXT: ret ptr [[P2]]
+;
+; LOWER-LABEL: define ptr @negative_operands_sext_chain(
+; LOWER-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; LOWER-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
+; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i32
+; LOWER-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
+; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP2]]
+; LOWER-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 -128
+; LOWER-NEXT: ret ptr [[UGLYGEP2]]
+;
+ %negative = or i8 %x, -128
+ %sum = add i8 %negative, -128
+ %extended = sext i8 %sum to i32
+ %index = sext i32 %extended to i64
+ %p = getelementptr i8, ptr %base, i64 %index
+ ret ptr %p
+}
+
+; The outer add makes the GEP index nonnegative, but that fact does not apply
+; to the inner add. Extract 128 without distributing sext through the inner
+; add: for x = 127, the original offset is 0, not 256.
+define ptr @nonnegative_outer_add(ptr %base, i8 %x) {
+; SPLIT-LABEL: define ptr @nonnegative_outer_add(
+; SPLIT-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; SPLIT-NEXT: [[SUM:%.*]] = add i8 [[X]], 1
+; SPLIT-NEXT: [[EXTENDED:%.*]] = sext i8 [[SUM]] to i64
+; SPLIT-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[EXTENDED]]
+; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP1]], i64 128
+; SPLIT-NEXT: ret ptr [[P2]]
+;
+; LOWER-LABEL: define ptr @nonnegative_outer_add(
+; LOWER-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; LOWER-NEXT: [[SUM:%.*]] = add i8 [[X]], 1
+; LOWER-NEXT: [[EXTENDED:%.*]] = sext i8 [[SUM]] to i64
+; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[EXTENDED]]
+; LOWER-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 128
+; LOWER-NEXT: ret ptr [[UGLYGEP2]]
+;
+ %sum = add i8 %x, 1
+ %extended = sext i8 %sum to i64
+ %index = add nsw i64 %extended, 128
+ %p = getelementptr i8, ptr %base, i64 %index
+ ret ptr %p
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
>From 6a19aef24e1958250646c386831049baad7d9034 Mon Sep 17 00:00:00 2001
From: Alex Maclean <amaclean at nvidia.com>
Date: Fri, 2 Oct 2026 12:35:21 -0700
Subject: [PATCH 2/3] [SeparateConstOffsetFromGEP] Fix sext/add overflow checks
---
.../Scalar/SeparateConstOffsetFromGEP.cpp | 157 +++++++++---------
.../NVPTX/sext-add-inbounds.ll | 51 +++---
.../NVPTX/sext-add-nonnegative.ll | 71 +++-----
3 files changed, 121 insertions(+), 158 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index 6e68c8a7a5e4d..054bde693e896 100644
--- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -167,6 +167,7 @@
#include "llvm/Transforms/Utils/Local.h"
#include <cassert>
#include <cstdint>
+#include <iterator>
#include <optional>
#include <string>
@@ -206,19 +207,18 @@ class ConstantOffsetExtractor {
/// Extracts a constant offset from the given GEP index. It returns the
/// new index representing the remainder (equal to the original index minus
/// the constant offset), or nullptr if we cannot extract a constant offset.
- /// \p Idx The given GEP index
- /// \p GEP The given GEP
+ /// \p Idx The given GEP index use
/// \p UserChainTail Outputs the tail of UserChain so that we can
/// garbage-collect unused instructions in UserChain.
/// \p PreservesNUW Outputs whether the extraction allows preserving the
/// GEP's nuw flag, if it has one.
- static Value *Extract(Value *Idx, GetElementPtrInst *GEP,
- User *&UserChainTail, bool &PreservesNUW);
+ static Value *Extract(const Use &Idx, User *&UserChainTail,
+ bool &PreservesNUW);
/// Looks for a constant offset from the given GEP index without extracting
/// it. It returns the numeric value of the extracted constant offset, or
/// std::nullopt on failure. The arguments have the same meaning as Extract.
- static std::optional<APInt> Find(Value *Idx, GetElementPtrInst *GEP);
+ static std::optional<APInt> Find(const Use &Idx);
private:
ConstantOffsetExtractor(BasicBlock::iterator InsertionPt)
@@ -229,16 +229,14 @@ class ConstantOffsetExtractor {
/// successful, returns C and update UserChain as a def-use chain from C to V;
/// otherwise, returns std::nullopt and UserChain is empty.
/// \p V The given expression
- /// \p GEP The base GEP instruction, used for determining relevant
- /// types, flags, and non-negativity needed for safe
- /// reassociation
- /// \p Idx The original index of the GEP
+ /// \p Idx The original index use of the GEP, or nullptr if its
+ /// sign and bounds information no longer applies
/// \p SignExtended Whether V will be sign-extended in the computation of
/// the GEP index
/// \p ZeroExtended Whether V will be zero-extended in the computation of
/// the GEP index
- std::optional<APInt> find(Value *V, GetElementPtrInst *GEP, Value *Idx,
- bool SignExtended, bool ZeroExtended);
+ std::optional<APInt> find(Value *V, const Use *Idx, bool SignExtended,
+ bool ZeroExtended);
/// A helper function to look into both operands of a binary operator.
std::optional<APInt>
@@ -292,11 +290,10 @@ class ConstantOffsetExtractor {
///
/// \p SignExtended Whether BO is surrounded by sext
/// \p ZeroExtended Whether BO is surrounded by zext
- /// \p GEP The base GEP instruction, used for determining relevant
- /// types and flags needed for safe reassociation.
- /// \p Idx The original index of the GEP
+ /// \p Idx The original index use of the GEP, or nullptr if its
+ /// sign and bounds information no longer applies
bool canTraceInto(bool SignExtended, bool ZeroExtended, BinaryOperator *BO,
- GetElementPtrInst *GEP, Value *Idx);
+ const Use *Idx);
/// Analyze a xor expression, and identify the bits in the constant operand
/// that are disjoint from the base operand's known set bits. For these
@@ -510,15 +507,15 @@ FunctionPass *llvm::createSeparateConstOffsetFromGEPPass(bool LowerGEP) {
// Checks if it is safe to reorder an add/sext result used in a GEP.
//
// An inbounds GEP does not guarantee that the index is non-negative.
-// This helper checks first if the index is known non-negative. If the index is
-// non-negative, the transform is always safe.
+// This helper checks first if the index and at least one add operand are known
+// non-negative. If so, the transform is safe.
// Second, it checks whether the GEP is inbounds and directly based on a global
// or an alloca, which are required to prove futher transform validity.
// If the GEP:
-// - Has a zero offset from the base, the index is non-negative (any negative
-// value would produce poison/UB)
+// - Has a zero offset from the base and Idx is its first index, the index is
+// non-negative (any negative value would produce poison/UB)
// - Has ObjectSize < (2^(N-1) - C + 1) * stride, where C is a constant from the
-// add, stride is the element size of Idx, and N is bitwidth of Idx.
+// add, stride is the element size of Idx, and N is bitwidth of the add.
// This is because with this pattern:
// %add = add iN %val, C
// %sext = sext iN %add to i64
@@ -537,12 +534,14 @@ FunctionPass *llvm::createSeparateConstOffsetFromGEPPass(bool LowerGEP) {
// producing a large positive value that still needs to be inbounds to the
// object size. If C is negative, we cannot make any useful assumptions based
// on the offset, since it would need to be extremely large.
-static bool canReorderAddSextToGEP(const GetElementPtrInst *GEP,
- const Value *Idx, const BinaryOperator *Add,
+static bool canReorderAddSextToGEP(const Use *Idx, const BinaryOperator *Add,
const DataLayout &DL) {
- if (isKnownNonNegative(Idx, DL))
+ if (isKnownNonNegative(Idx->get(), DL) &&
+ (isKnownNonNegative(Add->getOperand(0), DL) ||
+ isKnownNonNegative(Add->getOperand(1), DL)))
return true;
+ const auto *GEP = cast<GetElementPtrInst>(Idx->getUser());
if (!GEP->isInBounds())
return false;
@@ -561,7 +560,10 @@ static bool canReorderAddSextToGEP(const GetElementPtrInst *GEP,
// Calculate the threshold
APInt Threshold;
unsigned N = Add->getType()->getIntegerBitWidth();
- TypeSize ElemSize = DL.getTypeAllocSize(GEP->getSourceElementType());
+ auto GTI = gep_type_begin(GEP);
+ // Track the use: the same value may index different types in this GEP.
+ std::advance(GTI, Idx->getOperandNo() - 1);
+ TypeSize ElemSize = GTI.getSequentialElementStride(DL);
if (ElemSize.isScalable())
return false;
uint64_t Stride = ElemSize.getFixedValue();
@@ -577,8 +579,10 @@ static bool canReorderAddSextToGEP(const GetElementPtrInst *GEP,
APInt(128, Stride);
}
- if (Base && (isa<AllocaInst>(Base) || isa<GlobalObject>(Base)) &&
- !CI->isNegative()) {
+ // Only the first index is relative to Ptr. Earlier indices may move the
+ // pointer within the object, so later indices must use the object-size proof.
+ if (GTI == gep_type_begin(GEP) && Base &&
+ (isa<AllocaInst>(Base) || isa<GlobalObject>(Base)) && !CI->isNegative()) {
// If the offset is zero from an alloca or global, inbounds is sufficient to
// prove non-negativity if one add operand is non-negative
if (Offset == 0)
@@ -616,21 +620,17 @@ static bool canReorderAddSextToGEP(const GetElementPtrInst *GEP,
}
bool ConstantOffsetExtractor::canTraceInto(bool SignExtended, bool ZeroExtended,
- BinaryOperator *BO,
- GetElementPtrInst *GEP, Value *Idx) {
- // We only consider ADD, SUB and OR, because a non-zero constant found in
+ BinaryOperator *BO, const Use *Idx) {
+ // Do not trace into "or" unless it is equivalent to "add nuw nsw".
+ // This is the case if the or's disjoint flag is set.
+ if (BO->getOpcode() == Instruction::Or)
+ return cast<PossiblyDisjointInst>(BO)->isDisjoint();
+
+ // We only consider ADD and SUB here, because a non-zero constant found in
// expressions composed of these operations can be easily hoisted as a
// constant offset by reassociation.
if (BO->getOpcode() != Instruction::Add &&
- BO->getOpcode() != Instruction::Sub &&
- BO->getOpcode() != Instruction::Or) {
- return false;
- }
-
- // Do not trace into "or" unless it is equivalent to "add nuw nsw".
- // This is the case if the or's disjoint flag is set.
- if (BO->getOpcode() == Instruction::Or &&
- !cast<PossiblyDisjointInst>(BO)->isDisjoint())
+ BO->getOpcode() != Instruction::Sub)
return false;
// FIXME: We don't currently support constants from the RHS of subs,
@@ -650,7 +650,14 @@ bool ConstantOffsetExtractor::canTraceInto(bool SignExtended, bool ZeroExtended,
// 1 | 0 | sext(BO) == sext(A) op sext(B)
// 1 | 1 | zext(sext(BO)) ==
// | | zext(sext(A)) op zext(sext(B))
- if (BO->getOpcode() == Instruction::Add && !ZeroExtended && GEP) {
+ if (BO->getOpcode() == Instruction::Add && !ZeroExtended && Idx) {
+ const auto *GEP = cast<GetElementPtrInst>(Idx->getUser());
+ // For a sext(add nuw), allow tracing through when the enclosing GEP is both
+ // inbounds and nuw.
+ if (SignExtended && BO->hasNoUnsignedWrap() && GEP->isInBounds() &&
+ GEP->hasNoUnsignedWrap())
+ return true;
+
// If a + b >= 0 and (a >= 0 or b >= 0), then
// sext(a + b) = sext(a) + sext(b)
// even if the addition is not marked nsw.
@@ -659,27 +666,16 @@ bool ConstantOffsetExtractor::canTraceInto(bool SignExtended, bool ZeroExtended,
// index under certain conditions (see canReorderAddSextToGEP).
//
// Verified in @sext_add in split-gep.ll.
- if (canReorderAddSextToGEP(GEP, Idx, BO, DL))
+ if (canReorderAddSextToGEP(Idx, BO, DL))
return true;
}
- // For a sext(add nuw), allow tracing through when the enclosing GEP is both
- // inbounds and nuw.
- bool GEPInboundsNUW =
- GEP ? (GEP->isInBounds() && GEP->hasNoUnsignedWrap()) : false;
- if (BO->getOpcode() == Instruction::Add && SignExtended && !ZeroExtended &&
- GEPInboundsNUW && BO->hasNoUnsignedWrap())
- return true;
-
// sext (add/sub nsw A, B) == add/sub nsw (sext A), (sext B)
// zext (add/sub nuw A, B) == add/sub nuw (zext A), (zext B)
- if (BO->getOpcode() == Instruction::Add ||
- BO->getOpcode() == Instruction::Sub) {
- if (SignExtended && !BO->hasNoSignedWrap())
- return false;
- if (ZeroExtended && !BO->hasNoUnsignedWrap())
- return false;
- }
+ if (SignExtended && !BO->hasNoSignedWrap())
+ return false;
+ if (ZeroExtended && !BO->hasNoUnsignedWrap())
+ return false;
return true;
}
@@ -689,9 +685,10 @@ std::optional<APInt> ConstantOffsetExtractor::findInEitherOperand(
// Save off the current height of the chain, in case we need to restore it.
size_t ChainLength = UserChain.size();
- // BO cannot use information from the base GEP at this point, so clear it.
+ // An intervening binary operator invalidates the GEP's index sign and bounds
+ // information, so do not pass it to either operand.
std::optional<APInt> ConstantOffset =
- find(BO->getOperand(0), nullptr, nullptr, SignExtended, ZeroExtended);
+ find(BO->getOperand(0), nullptr, SignExtended, ZeroExtended);
// If we found a constant offset in the left operand, stop and return that.
// This shortcut might cause us to miss opportunities of combining the
// constant offsets in both operands, e.g., (a + 4) + (b + 5) => (a + b) + 9.
@@ -704,8 +701,7 @@ std::optional<APInt> ConstantOffsetExtractor::findInEitherOperand(
// since visiting the LHS didn't pan out.
UserChain.resize(ChainLength);
- ConstantOffset =
- find(BO->getOperand(1), nullptr, nullptr, SignExtended, ZeroExtended);
+ ConstantOffset = find(BO->getOperand(1), nullptr, SignExtended, ZeroExtended);
// If U is a sub operator, negate the constant offset found in the right
// operand.
if (ConstantOffset && BO->getOpcode() == Instruction::Sub)
@@ -718,9 +714,9 @@ std::optional<APInt> ConstantOffsetExtractor::findInEitherOperand(
return ConstantOffset;
}
-std::optional<APInt>
-ConstantOffsetExtractor::find(Value *V, GetElementPtrInst *GEP, Value *Idx,
- bool SignExtended, bool ZeroExtended) {
+std::optional<APInt> ConstantOffsetExtractor::find(Value *V, const Use *Idx,
+ bool SignExtended,
+ bool ZeroExtended) {
// TODO(jingyue): We could trace into integer/pointer casts, such as
// inttoptr, ptrtoint, bitcast, and addrspacecast. We choose to handle only
// integers because it gives good enough results for our benchmarks.
@@ -740,26 +736,25 @@ ConstantOffsetExtractor::find(Value *V, GetElementPtrInst *GEP, Value *Idx,
ConstantOffset = CI->getValue();
} else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(V)) {
// Trace into subexpressions for more hoisting opportunities.
- if (canTraceInto(SignExtended, ZeroExtended, BO, GEP, Idx))
+ if (canTraceInto(SignExtended, ZeroExtended, BO, Idx))
ConstantOffset = findInEitherOperand(BO, SignExtended, ZeroExtended);
else if (BO->getOpcode() == Instruction::Xor)
ConstantOffset = extractDisjointBitsFromXor(BO);
} else if (isa<TruncInst>(V)) {
if (SignExtended || ZeroExtended)
return ConstantOffset;
- ConstantOffset =
- find(U->getOperand(0), GEP, Idx, SignExtended, ZeroExtended);
+ ConstantOffset = find(U->getOperand(0), Idx, SignExtended, ZeroExtended);
if (ConstantOffset)
*ConstantOffset = ConstantOffset->trunc(BitWidth);
} else if (isa<SExtInst>(V)) {
ConstantOffset =
- find(U->getOperand(0), GEP, Idx, /* SignExtended */ true, ZeroExtended);
+ find(U->getOperand(0), Idx, /* SignExtended */ true, ZeroExtended);
if (ConstantOffset)
*ConstantOffset = ConstantOffset->sext(BitWidth);
} else if (isa<ZExtInst>(V)) {
// As an optimization, we can clear the SignExtended flag because
// sext(zext(a)) = zext(a). Verified in @sext_zext in split-gep.ll.
- ConstantOffset = find(U->getOperand(0), GEP, Idx, /* SignExtended */ false,
+ ConstantOffset = find(U->getOperand(0), Idx, /* SignExtended */ false,
/* ZeroExtended */ true);
if (ConstantOffset)
*ConstantOffset = ConstantOffset->zext(BitWidth);
@@ -980,20 +975,18 @@ static bool allowsPreservingNUW(const User *U) {
return true;
}
-static BasicBlock::iterator getIndexInsertionPoint(Value *Idx,
- GetElementPtrInst *GEP) {
- if (auto *I = dyn_cast<Instruction>(Idx))
+static BasicBlock::iterator getIndexInsertionPoint(const Use &Idx) {
+ if (auto *I = dyn_cast<Instruction>(Idx.get()))
if (auto IP = I->getInsertionPointAfterDef())
return *IP;
- return GEP->getIterator();
+ return cast<GetElementPtrInst>(Idx.getUser())->getIterator();
}
-Value *ConstantOffsetExtractor::Extract(Value *Idx, GetElementPtrInst *GEP,
- User *&UserChainTail,
+Value *ConstantOffsetExtractor::Extract(const Use &Idx, User *&UserChainTail,
bool &PreservesNUW) {
- ConstantOffsetExtractor Extractor(getIndexInsertionPoint(Idx, GEP));
+ ConstantOffsetExtractor Extractor(getIndexInsertionPoint(Idx));
// Find a constant offset first.
- if (!Extractor.find(Idx, GEP, Idx, /* SignExtended */ false,
+ if (!Extractor.find(Idx, &Idx, /* SignExtended */ false,
/* ZeroExtended */ false)) {
UserChainTail = nullptr;
PreservesNUW = true;
@@ -1008,10 +1001,10 @@ Value *ConstantOffsetExtractor::Extract(Value *Idx, GetElementPtrInst *GEP,
return IdxWithoutConstOffset;
}
-std::optional<APInt> ConstantOffsetExtractor::Find(Value *Idx,
- GetElementPtrInst *GEP) {
+std::optional<APInt> ConstantOffsetExtractor::Find(const Use &Idx) {
+ auto *GEP = cast<GetElementPtrInst>(Idx.getUser());
return ConstantOffsetExtractor(GEP->getIterator())
- .find(Idx, GEP, Idx, /* SignExtended */ false, /* ZeroExtended */ false);
+ .find(Idx, &Idx, /* SignExtended */ false, /* ZeroExtended */ false);
}
bool SeparateConstOffsetFromGEP::canonicalizeArrayIndicesToIndexSize(
@@ -1025,7 +1018,7 @@ bool SeparateConstOffsetFromGEP::canonicalizeArrayIndicesToIndexSize(
if (GTI.isSequential()) {
if ((*I)->getType() != PtrIdxTy) {
*I = CastInst::CreateIntegerCast(*I, PtrIdxTy, true, "idxprom",
- getIndexInsertionPoint(*I, GEP));
+ getIndexInsertionPoint(*I));
Changed = true;
}
}
@@ -1049,7 +1042,7 @@ APInt SeparateConstOffsetFromGEP::accumulateByteOffset(GetElementPtrInst *GEP,
// Tries to extract a constant offset from this GEP index.
if (std::optional<APInt> ConstantOffset =
- ConstantOffsetExtractor::Find(GEP->getOperand(I), GEP)) {
+ ConstantOffsetExtractor::Find(GEP->getOperandUse(I))) {
NeedsExtraction = true;
// A GEP may have multiple indices. We accumulate the extracted
// constant offset to a byte offset, and later offset the remainder of
@@ -1281,8 +1274,8 @@ bool SeparateConstOffsetFromGEP::splitGEP(GetElementPtrInst *GEP) {
Value *Idx = GEP->getOperand(I);
User *UserChainTail;
bool PreservesNUW;
- Value *NewIdx = ConstantOffsetExtractor::Extract(Idx, GEP, UserChainTail,
- PreservesNUW);
+ Value *NewIdx = ConstantOffsetExtractor::Extract(
+ GEP->getOperandUse(I), UserChainTail, PreservesNUW);
if (NewIdx != nullptr) {
// Switches to the index with the constant offset removed.
GEP->setOperand(I, NewIdx);
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-inbounds.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-inbounds.ll
index 6af5f7c977557..566650e1f83be 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-inbounds.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-inbounds.ll
@@ -12,16 +12,18 @@
define ptr @preceding_index(i8 %x) {
; SPLIT-LABEL: define ptr @preceding_index(
; SPLIT-SAME: i8 [[X:%.*]]) {
-; SPLIT-NEXT: [[INDEX:%.*]] = sext i8 [[X]] to i64
+; SPLIT-NEXT: [[SUM:%.*]] = add i8 [[X]], 1
+; SPLIT-NEXT: [[INDEX:%.*]] = sext i8 [[SUM]] to i64
; SPLIT-NEXT: [[TMP1:%.*]] = getelementptr [128 x i8], ptr @array, i64 0, i64 [[INDEX]]
-; SPLIT-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[TMP1]], i64 129
+; SPLIT-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[TMP1]], i64 128
; SPLIT-NEXT: ret ptr [[P1]]
;
; LOWER-LABEL: define ptr @preceding_index(
; LOWER-SAME: i8 [[X:%.*]]) {
-; LOWER-NEXT: [[INDEX:%.*]] = sext i8 [[X]] to i64
+; LOWER-NEXT: [[SUM:%.*]] = add i8 [[X]], 1
+; LOWER-NEXT: [[INDEX:%.*]] = sext i8 [[SUM]] to i64
; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr @array, i64 [[INDEX]]
-; LOWER-NEXT: [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 129
+; LOWER-NEXT: [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 128
; LOWER-NEXT: ret ptr [[UGLYGEP1]]
;
%sum = add i8 %x, 1
@@ -33,19 +35,12 @@ define ptr @preceding_index(i8 %x) {
; The second index has stride 1, not 256. For x = -128, signed underflow gives
; a valid offset of 127 into this object. Distributing sext would give -129.
define ptr @negative_offset(i8 %x) {
-; SPLIT-LABEL: define ptr @negative_offset(
-; SPLIT-SAME: i8 [[X:%.*]]) {
-; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[X]] to i64
-; SPLIT-NEXT: [[TMP2:%.*]] = getelementptr [256 x i8], ptr @array, i64 0, i64 [[TMP1]]
-; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -1
-; SPLIT-NEXT: ret ptr [[P2]]
-;
-; LOWER-LABEL: define ptr @negative_offset(
-; LOWER-SAME: i8 [[X:%.*]]) {
-; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[X]] to i64
-; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr @array, i64 [[TMP1]]
-; LOWER-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 -1
-; LOWER-NEXT: ret ptr [[UGLYGEP2]]
+; CHECK-LABEL: define ptr @negative_offset(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT: [[SUM:%.*]] = add i8 [[X]], -1
+; CHECK-NEXT: [[INDEX:%.*]] = sext i8 [[SUM]] to i64
+; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds [256 x i8], ptr @array, i64 0, i64 [[INDEX]]
+; CHECK-NEXT: ret ptr [[P]]
;
%sum = add i8 %x, -1
%index = sext i8 %sum to i64
@@ -59,16 +54,18 @@ define ptr @negative_offset(i8 %x) {
define ptr @interior_base(i8 %x) {
; SPLIT-LABEL: define ptr @interior_base(
; SPLIT-SAME: i8 [[X:%.*]]) {
-; SPLIT-NEXT: [[INDEX:%.*]] = sext i8 [[X]] to i64
+; SPLIT-NEXT: [[SUM:%.*]] = add i8 [[X]], 1
+; SPLIT-NEXT: [[INDEX:%.*]] = sext i8 [[SUM]] to i64
; SPLIT-NEXT: [[TMP1:%.*]] = getelementptr [256 x i8], ptr @array, i64 0, i64 [[INDEX]]
-; SPLIT-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[TMP1]], i64 129
+; SPLIT-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[TMP1]], i64 128
; SPLIT-NEXT: ret ptr [[P1]]
;
; LOWER-LABEL: define ptr @interior_base(
; LOWER-SAME: i8 [[X:%.*]]) {
-; LOWER-NEXT: [[INDEX:%.*]] = sext i8 [[X]] to i64
+; LOWER-NEXT: [[SUM:%.*]] = add i8 [[X]], 1
+; LOWER-NEXT: [[INDEX:%.*]] = sext i8 [[SUM]] to i64
; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr @array, i64 [[INDEX]]
-; LOWER-NEXT: [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 129
+; LOWER-NEXT: [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 128
; LOWER-NEXT: ret ptr [[UGLYGEP1]]
;
%base = getelementptr i8, ptr @array, i64 128
@@ -85,20 +82,22 @@ define ptr @interior_base(i8 %x) {
define ptr @repeated_index(i8 %x) {
; SPLIT-LABEL: define ptr @repeated_index(
; SPLIT-SAME: i8 [[X:%.*]]) {
-; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[X]] to i64
+; SPLIT-NEXT: [[SUM:%.*]] = add i8 [[X]], -1
+; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[SUM]] to i64
; SPLIT-NEXT: [[TMP2:%.*]] = sext i8 [[X]] to i64
; SPLIT-NEXT: [[TMP3:%.*]] = getelementptr [256 x i8], ptr @array, i64 [[TMP2]], i64 [[TMP1]]
-; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP3]], i64 -257
+; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP3]], i64 -256
; SPLIT-NEXT: ret ptr [[P2]]
;
; LOWER-LABEL: define ptr @repeated_index(
; LOWER-SAME: i8 [[X:%.*]]) {
-; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[X]] to i64
+; LOWER-NEXT: [[SUM:%.*]] = add i8 [[X]], -1
+; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[SUM]] to i64
; LOWER-NEXT: [[TMP3:%.*]] = sext i8 [[X]] to i64
; LOWER-NEXT: [[TMP2:%.*]] = shl i64 [[TMP3]], 8
; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr @array, i64 [[TMP2]]
; LOWER-NEXT: [[UGLYGEP4:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 [[TMP1]]
-; LOWER-NEXT: [[UGLYGEP3:%.*]] = getelementptr i8, ptr [[UGLYGEP4]], i64 -257
+; LOWER-NEXT: [[UGLYGEP3:%.*]] = getelementptr i8, ptr [[UGLYGEP4]], i64 -256
; LOWER-NEXT: ret ptr [[UGLYGEP3]]
;
%sum = add i8 %x, -1
@@ -106,5 +105,3 @@ define ptr @repeated_index(i8 %x) {
%p = getelementptr inbounds [256 x i8], ptr @array, i64 %index, i64 %index
ret ptr %p
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-nonnegative.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-nonnegative.ll
index efb27da9ff9bb..c7ea81e4ab19b 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-nonnegative.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-nonnegative.ll
@@ -5,21 +5,13 @@
; A nonnegative sum of two negative operands may have signed overflow.
; For x = 0, the original index is 0, but distributing sext would give -256.
define ptr @negative_operands(ptr %base, i8 %x) {
-; SPLIT-LABEL: define ptr @negative_operands(
-; SPLIT-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
-; SPLIT-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
-; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i64
-; SPLIT-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
-; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -128
-; SPLIT-NEXT: ret ptr [[P2]]
-;
-; LOWER-LABEL: define ptr @negative_operands(
-; LOWER-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
-; LOWER-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
-; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i64
-; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
-; LOWER-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 -128
-; LOWER-NEXT: ret ptr [[UGLYGEP2]]
+; CHECK-LABEL: define ptr @negative_operands(
+; CHECK-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; CHECK-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
+; CHECK-NEXT: [[SUM:%.*]] = add i8 [[NEGATIVE]], -128
+; CHECK-NEXT: [[INDEX:%.*]] = sext i8 [[SUM]] to i64
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[INDEX]]
+; CHECK-NEXT: ret ptr [[P]]
;
%negative = or i8 %x, -128
%sum = add i8 %negative, -128
@@ -29,21 +21,13 @@ define ptr @negative_operands(ptr %base, i8 %x) {
}
define ptr @negative_operands_commuted(ptr %base, i8 %x) {
-; SPLIT-LABEL: define ptr @negative_operands_commuted(
-; SPLIT-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
-; SPLIT-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
-; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i64
-; SPLIT-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
-; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -128
-; SPLIT-NEXT: ret ptr [[P2]]
-;
-; LOWER-LABEL: define ptr @negative_operands_commuted(
-; LOWER-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
-; LOWER-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
-; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i64
-; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
-; LOWER-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 -128
-; LOWER-NEXT: ret ptr [[UGLYGEP2]]
+; CHECK-LABEL: define ptr @negative_operands_commuted(
+; CHECK-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; CHECK-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
+; CHECK-NEXT: [[SUM:%.*]] = add i8 -128, [[NEGATIVE]]
+; CHECK-NEXT: [[INDEX:%.*]] = sext i8 [[SUM]] to i64
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[INDEX]]
+; CHECK-NEXT: ret ptr [[P]]
;
%negative = or i8 %x, -128
%sum = add i8 -128, %negative
@@ -110,23 +94,14 @@ define ptr @nonnegative_rhs(ptr %base, i8 %x) {
; Recursing through sexts must still check the sign of an add operand.
define ptr @negative_operands_sext_chain(ptr %base, i8 %x) {
-; SPLIT-LABEL: define ptr @negative_operands_sext_chain(
-; SPLIT-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
-; SPLIT-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
-; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i32
-; SPLIT-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
-; SPLIT-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP2]]
-; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP3]], i64 -128
-; SPLIT-NEXT: ret ptr [[P2]]
-;
-; LOWER-LABEL: define ptr @negative_operands_sext_chain(
-; LOWER-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
-; LOWER-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
-; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i32
-; LOWER-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
-; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP2]]
-; LOWER-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 -128
-; LOWER-NEXT: ret ptr [[UGLYGEP2]]
+; CHECK-LABEL: define ptr @negative_operands_sext_chain(
+; CHECK-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; CHECK-NEXT: [[NEGATIVE:%.*]] = or i8 [[X]], -128
+; CHECK-NEXT: [[SUM:%.*]] = add i8 [[NEGATIVE]], -128
+; CHECK-NEXT: [[EXTENDED:%.*]] = sext i8 [[SUM]] to i32
+; CHECK-NEXT: [[INDEX:%.*]] = sext i32 [[EXTENDED]] to i64
+; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[INDEX]]
+; CHECK-NEXT: ret ptr [[P]]
;
%negative = or i8 %x, -128
%sum = add i8 %negative, -128
@@ -162,5 +137,3 @@ define ptr @nonnegative_outer_add(ptr %base, i8 %x) {
%p = getelementptr i8, ptr %base, i64 %index
ret ptr %p
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
>From 0c53b555a409797e788f5abdf5ef5088c9999de0 Mon Sep 17 00:00:00 2001
From: Alex Maclean <amaclean at nvidia.com>
Date: Mon, 5 Oct 2026 08:53:17 -0700
Subject: [PATCH 3/3] [SeparateConstOffsetFromGEP] Address review feedback
---
.../Scalar/SeparateConstOffsetFromGEP.cpp | 22 +++++--------
.../NVPTX/sext-add-nonnegative.ll | 31 ++++++++++++++++++-
2 files changed, 38 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index 054bde693e896..caf9656e58298 100644
--- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -507,8 +507,8 @@ FunctionPass *llvm::createSeparateConstOffsetFromGEPPass(bool LowerGEP) {
// Checks if it is safe to reorder an add/sext result used in a GEP.
//
// An inbounds GEP does not guarantee that the index is non-negative.
-// This helper checks first if the index and at least one add operand are known
-// non-negative. If so, the transform is safe.
+// This helper first checks whether value tracking proves that the add cannot
+// have signed overflow. If so, the transform is safe.
// Second, it checks whether the GEP is inbounds and directly based on a global
// or an alloca, which are required to prove futher transform validity.
// If the GEP:
@@ -536,9 +536,8 @@ FunctionPass *llvm::createSeparateConstOffsetFromGEPPass(bool LowerGEP) {
// on the offset, since it would need to be extremely large.
static bool canReorderAddSextToGEP(const Use *Idx, const BinaryOperator *Add,
const DataLayout &DL) {
- if (isKnownNonNegative(Idx->get(), DL) &&
- (isKnownNonNegative(Add->getOperand(0), DL) ||
- isKnownNonNegative(Add->getOperand(1), DL)))
+ if (computeOverflowForSignedAdd(cast<AddOperator>(Add), DL) ==
+ OverflowResult::NeverOverflows)
return true;
const auto *GEP = cast<GetElementPtrInst>(Idx->getUser());
@@ -560,9 +559,8 @@ static bool canReorderAddSextToGEP(const Use *Idx, const BinaryOperator *Add,
// Calculate the threshold
APInt Threshold;
unsigned N = Add->getType()->getIntegerBitWidth();
- auto GTI = gep_type_begin(GEP);
// Track the use: the same value may index different types in this GEP.
- std::advance(GTI, Idx->getOperandNo() - 1);
+ auto GTI = std::next(gep_type_begin(GEP), Idx->getOperandNo() - 1);
TypeSize ElemSize = GTI.getSequentialElementStride(DL);
if (ElemSize.isScalable())
return false;
@@ -581,7 +579,7 @@ static bool canReorderAddSextToGEP(const Use *Idx, const BinaryOperator *Add,
// Only the first index is relative to Ptr. Earlier indices may move the
// pointer within the object, so later indices must use the object-size proof.
- if (GTI == gep_type_begin(GEP) && Base &&
+ if (Idx->getOperandNo() == 1 && Base &&
(isa<AllocaInst>(Base) || isa<GlobalObject>(Base)) && !CI->isNegative()) {
// If the offset is zero from an alloca or global, inbounds is sufficient to
// prove non-negativity if one add operand is non-negative
@@ -658,12 +656,8 @@ bool ConstantOffsetExtractor::canTraceInto(bool SignExtended, bool ZeroExtended,
GEP->hasNoUnsignedWrap())
return true;
- // If a + b >= 0 and (a >= 0 or b >= 0), then
- // sext(a + b) = sext(a) + sext(b)
- // even if the addition is not marked nsw.
- //
- // Leveraging this invariant, we can trace into an sext'ed inbound GEP
- // index under certain conditions (see canReorderAddSextToGEP).
+ // Trace through sext when value tracking or the GEP's bounds prove that
+ // the addition cannot have signed overflow.
//
// Verified in @sext_add in split-gep.ll.
if (canReorderAddSextToGEP(Idx, BO, DL))
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-nonnegative.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-nonnegative.ll
index c7ea81e4ab19b..29869d59f1d57 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-nonnegative.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/sext-add-nonnegative.ll
@@ -92,7 +92,36 @@ define ptr @nonnegative_rhs(ptr %base, i8 %x) {
ret ptr %p
}
-; Recursing through sexts must still check the sign of an add operand.
+; Both operands and the sum are negative, but the sum is in [-96, -65], so
+; value tracking can prove that the addition cannot have signed overflow.
+define ptr @negative_sum_no_overflow(ptr %base, i8 %x) {
+; SPLIT-LABEL: define ptr @negative_sum_no_overflow(
+; SPLIT-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; SPLIT-NEXT: [[MASKED:%.*]] = and i8 [[X]], 31
+; SPLIT-NEXT: [[NEGATIVE:%.*]] = or i8 [[MASKED]], -64
+; SPLIT-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i64
+; SPLIT-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
+; SPLIT-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -32
+; SPLIT-NEXT: ret ptr [[P2]]
+;
+; LOWER-LABEL: define ptr @negative_sum_no_overflow(
+; LOWER-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
+; LOWER-NEXT: [[MASKED:%.*]] = and i8 [[X]], 31
+; LOWER-NEXT: [[NEGATIVE:%.*]] = or i8 [[MASKED]], -64
+; LOWER-NEXT: [[TMP1:%.*]] = sext i8 [[NEGATIVE]] to i64
+; LOWER-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP1]]
+; LOWER-NEXT: [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[UGLYGEP]], i64 -32
+; LOWER-NEXT: ret ptr [[UGLYGEP2]]
+;
+ %masked = and i8 %x, 31
+ %negative = or i8 %masked, -64
+ %sum = add i8 %negative, -32
+ %index = sext i8 %sum to i64
+ %p = getelementptr i8, ptr %base, i64 %index
+ ret ptr %p
+}
+
+; Recursing through sexts must still rule out signed overflow.
define ptr @negative_operands_sext_chain(ptr %base, i8 %x) {
; CHECK-LABEL: define ptr @negative_operands_sext_chain(
; CHECK-SAME: ptr [[BASE:%.*]], i8 [[X:%.*]]) {
More information about the llvm-commits
mailing list