[InstCombine] Canonicalize signed saturated additions #153053
+426
−8
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
llvmbot
added
llvm:instcombine
|
@llvm/pr-subscribers-llvm-transforms Author: AZero13 (AZero13) Changeshttps://alive2.llvm.org/ce/z/YGT5SN This is tricky because with positive numbers, we only go up, so we can in fact always hit the signed_max boundary. This is important because the intrinsic we use has the behavior of going the OTHER way, aka clamp to INT_MIN if it goes in that direction. And the range checking we do only works for positive numbers. Because of this issue, we can only do this for constants at the moment. Full diff: https://github.com/llvm/llvm-project/pull/153053.diff 3 Files Affected:
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index eb4332fbc0959..b67f667405a50 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1012,10 +1012,9 @@ static Value *canonicalizeSaturatedSubtract(const ICmpInst *ICI,
return Result;
}
-static Value *canonicalizeSaturatedAdd(ICmpInst *Cmp, Value *TVal, Value *FVal,
- InstCombiner::BuilderTy &Builder) {
- if (!Cmp->hasOneUse())
- return nullptr;
+static Value *
+canonicalizeSaturatedAddUnsigned(ICmpInst *Cmp, Value *TVal, Value *FVal,
+ InstCombiner::BuilderTy &Builder) {
// Match unsigned saturated add with constant.
Value *Cmp0 = Cmp->getOperand(0);
@@ -1037,8 +1036,7 @@ static Value *canonicalizeSaturatedAdd(ICmpInst *Cmp, Value *TVal, Value *FVal,
// uge -1 is canonicalized to eq -1 and requires special handling
// (a == -1) ? -1 : a + 1 -> uadd.sat(a, 1)
if (Pred == ICmpInst::ICMP_EQ) {
- if (match(FVal, m_Add(m_Specific(Cmp0), m_One())) &&
- match(Cmp1, m_AllOnes())) {
+ if (match(FVal, m_Add(m_Specific(Cmp0), m_One())) && Cmp1 == TVal) {
return Builder.CreateBinaryIntrinsic(
Intrinsic::uadd_sat, Cmp0, ConstantInt::get(Cmp0->getType(), 1));
}
@@ -1115,6 +1113,94 @@ static Value *canonicalizeSaturatedAdd(ICmpInst *Cmp, Value *TVal, Value *FVal,
return nullptr;
}
+static Value *canonicalizeSaturatedAddSigned(ICmpInst *Cmp, Value *TVal,
+ Value *FVal,
+ InstCombiner::BuilderTy &Builder) {
+ // Match saturated add with constant.
+ Value *Cmp0 = Cmp->getOperand(0);
+ Value *Cmp1 = Cmp->getOperand(1);
+ ICmpInst::Predicate Pred = Cmp->getPredicate();
+ const APInt *C;
+
+ // Canonicalize INT_MAX to true value of the select.
+ if (match(FVal, m_MaxSignedValue())) {
+ std::swap(TVal, FVal);
+ Pred = CmpInst::getInversePredicate(Pred);
+ }
+ if (!match(TVal, m_MaxSignedValue()))
+ return nullptr;
+
+ // sge maximum signed value is canonicalized to eq minimum signed value and
+ // requires special handling (a == INT_MAX) ? INT_MAX : a + 1 -> sadd.sat(a,
+ // 1)
+ if (Pred == ICmpInst::ICMP_EQ) {
+ if (match(FVal, m_Add(m_Specific(Cmp0), m_One())) && Cmp1 == TVal) {
+ return Builder.CreateBinaryIntrinsic(
+ Intrinsic::sadd_sat, Cmp0, ConstantInt::get(Cmp0->getType(), 1));
+ }
+ return nullptr;
+ }
+
+ if ((Pred == ICmpInst::ICMP_SGE || Pred == ICmpInst::ICMP_SGT) &&
+ match(FVal, m_Add(m_Specific(Cmp0), m_APIntAllowPoison(C))) &&
+ match(Cmp1, m_SpecificIntAllowPoison(
+ APInt::getSignedMaxValue(
+ Cmp1->getType()->getScalarSizeInBits()) -
+ *C)) &&
+ !C->isNegative()) {
+ // (X > INT_MAX - C) ? INT_MAX : (X + C) --> sadd.sat(X, C)
+ // (X >= INT_MAX - C) ? INT_MAX : (X + C) --> sadd.sat(X, C)
+ return Builder.CreateBinaryIntrinsic(Intrinsic::sadd_sat, Cmp0,
+ ConstantInt::get(Cmp0->getType(), *C));
+ }
+
+ if (Pred == ICmpInst::ICMP_SGT &&
+ match(FVal, m_Add(m_Specific(Cmp0), m_APIntAllowPoison(C))) &&
+ match(Cmp1, m_SpecificIntAllowPoison(
+ APInt::getSignedMaxValue(
+ Cmp1->getType()->getScalarSizeInBits()) -
+ *C - 1)) &&
+ !C->isNegative()) {
+ // (X > INT_MAX - C - 1) ? INT_MAX : (X + C) --> sadd.sat(X, C)
+ return Builder.CreateBinaryIntrinsic(Intrinsic::sadd_sat, Cmp0,
+ ConstantInt::get(Cmp0->getType(), *C));
+ }
+
+ if (Pred == ICmpInst::ICMP_SGE &&
+ match(FVal, m_Add(m_Specific(Cmp0), m_APIntAllowPoison(C))) &&
+ match(Cmp1, m_SpecificIntAllowPoison(
+ APInt::getSignedMinValue(
+ Cmp1->getType()->getScalarSizeInBits()) -
+ *C)) &&
+ !C->isNegative()) {
+ // (X >= INT_MAX - C + 1) ? INT_MAX : (X + C) --> sadd.sat(X, C)
+ return Builder.CreateBinaryIntrinsic(Intrinsic::sadd_sat, Cmp0,
+ ConstantInt::get(Cmp0->getType(), *C));
+ }
+
+ // TODO: Try to match variables. However, due to the fact that we can only
+ // fold if we know at least one is positive, we cannot fold for each and every
+ // time, unlike the unsigned case, where every number is positive.
+
+ // TODO: Match when known negatives go towards INT_MIN.
+
+ return nullptr;
+}
+
+static Value *canonicalizeSaturatedAdd(ICmpInst *Cmp, Value *TVal, Value *FVal,
+ InstCombiner::BuilderTy &Builder) {
+ if (!Cmp->hasOneUse())
+ return nullptr;
+
+ if (Value *V = canonicalizeSaturatedAddUnsigned(Cmp, TVal, FVal, Builder))
+ return V;
+
+ if (Value *V = canonicalizeSaturatedAddSigned(Cmp, TVal, FVal, Builder))
+ return V;
+
+ return nullptr;
+}
+
/// Try to match patterns with select and subtract as absolute difference.
static Value *foldAbsDiff(ICmpInst *Cmp, Value *TVal, Value *FVal,
InstCombiner::BuilderTy &Builder) {
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-const-to-bop.ll b/llvm/test/Transforms/InstCombine/canonicalize-const-to-bop.ll
index b3093a92624ae..f0e40f4ede161 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-const-to-bop.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-const-to-bop.ll
@@ -123,8 +123,7 @@ define i8 @udiv_slt_exact(i8 %x) {
define i8 @canonicalize_icmp_operands(i8 %x) {
; CHECK-LABEL: define i8 @canonicalize_icmp_operands(
; CHECK-SAME: i8 [[X:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.smin.i8(i8 [[X]], i8 119)
-; CHECK-NEXT: [[S:%.*]] = add nsw i8 [[TMP1]], 8
+; CHECK-NEXT: [[S:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X]], i8 8)
; CHECK-NEXT: ret i8 [[S]]
;
%add = add nsw i8 %x, 8
diff --git a/llvm/test/Transforms/InstCombine/saturating-add-sub.ll b/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
index cfd679c0cc592..6bfdfaa1a6288 100644
--- a/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
+++ b/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
@@ -2351,3 +2351,242 @@ define i8 @fold_add_umax_to_usub_multiuse(i8 %a) {
}
declare void @usei8(i8)
+
+define i8 @sadd_sat_uge_int_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_uge_int_max(
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[X:%.*]], 127
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP]], i8 127, i8 [[ADD]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %cmp = icmp sge i8 %x, 127
+ %add = add i8 %x, %y
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define i8 @sadd_sat_ugt_int_max(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_ugt_int_max(
+; CHECK-NEXT: [[R:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %cmp = icmp sgt i8 %x, 127
+ %add = add i8 %x, %y
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define i8 @sadd_sat_eq_int_max(i8 %x) {
+; CHECK-LABEL: @sadd_sat_eq_int_max(
+; CHECK-NEXT: [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 1)
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %cmp = icmp eq i8 %x, 127
+ %add = add i8 %x, 1
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define i8 @sadd_sat_constant(i8 %x) {
+; CHECK-LABEL: @sadd_sat_constant(
+; CHECK-NEXT: [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 10)
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %cmp = icmp sge i8 %x, 118
+ %add = add i8 %x, 10
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define i8 @sadd_sat_negative_no_fold(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_negative_no_fold(
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[X:%.*]], 127
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP]], i8 127, i8 [[ADD]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %cmp = icmp sge i8 %x, 127
+ %add = add i8 %x, %y
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define i8 @sadd_sat_wrong_predicate(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_wrong_predicate(
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i8 [[X:%.*]], 127
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[Y:%.*]], 127
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP_NOT]], i8 [[ADD]], i8 127
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %cmp = icmp slt i8 %x, 127
+ %add = add i8 %x, %y
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define i8 @sadd_sat_wrong_constant(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_wrong_constant(
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i8 [[X:%.*]], 125
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP]], i8 127, i8 [[ADD]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %cmp = icmp sge i8 %x, 126
+ %add = add i8 %x, %y
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define <2 x i8> @sadd_sat_vector(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @sadd_sat_vector(
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], splat (i8 127)
+; CHECK-NEXT: [[ADD:%.*]] = add <2 x i8> [[X]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = select <2 x i1> [[CMP]], <2 x i8> splat (i8 127), <2 x i8> [[ADD]]
+; CHECK-NEXT: ret <2 x i8> [[R]]
+;
+ %cmp = icmp sge <2 x i8> %x, <i8 127, i8 127>
+ %add = add <2 x i8> %x, %y
+ %r = select <2 x i1> %cmp, <2 x i8> <i8 127, i8 127>, <2 x i8> %add
+ ret <2 x i8> %r
+}
+
+define <2 x i8> @sadd_sat_vector_constant(<2 x i8> %x) {
+; CHECK-LABEL: @sadd_sat_vector_constant(
+; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 117, i8 107>)
+; CHECK-NEXT: [[R:%.*]] = add <2 x i8> [[TMP1]], <i8 10, i8 20>
+; CHECK-NEXT: ret <2 x i8> [[R]]
+;
+ %cmp = icmp sge <2 x i8> %x, <i8 118, i8 108>
+ %add = add <2 x i8> %x, <i8 10, i8 20>
+ %r = select <2 x i1> %cmp, <2 x i8> <i8 127, i8 127>, <2 x i8> %add
+ ret <2 x i8> %r
+}
+
+define i8 @sadd_sat_int_max_minus_x(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_int_max_minus_x(
+; CHECK-NEXT: [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %sub = sub i8 127, %x
+ %cmp = icmp slt i8 %sub, %y
+ %add = add i8 %x, %y
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define i8 @sadd_sat_int_max_minus_x_commuted(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_int_max_minus_x_commuted(
+; CHECK-NEXT: [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %sub = sub i8 127, %x
+ %cmp = icmp sgt i8 %y, %sub
+ %add = add i8 %x, %y
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define i8 @sadd_sat_int_max_minus_x_nonstrict(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_int_max_minus_x_nonstrict(
+; CHECK-NEXT: [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %sub = sub i8 127, %x
+ %cmp = icmp sle i8 %sub, %y
+ %add = add i8 %x, %y
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define i8 @sadd_sat_int_max_minus_x_commuted_nonstrict(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_int_max_minus_x_commuted_nonstrict(
+; CHECK-NEXT: [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %sub = sub i8 127, %x
+ %cmp = icmp sge i8 %y, %sub
+ %add = add i8 %x, %y
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define i8 @sadd_sat_int_max_minus_x_wrong_constant(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_int_max_minus_x_wrong_constant(
+; CHECK-NEXT: [[SUB:%.*]] = sub i8 126, [[X:%.*]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i8 [[SUB]], [[Y:%.*]]
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[X]], [[Y]]
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP]], i8 127, i8 [[ADD]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %sub = sub i8 126, %x
+ %cmp = icmp slt i8 %sub, %y
+ %add = add i8 %x, %y
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define i8 @sadd_sat_int_max_minus_x_wrong_predicate(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_int_max_minus_x_wrong_predicate(
+; CHECK-NEXT: [[SUB:%.*]] = sub i8 127, [[X:%.*]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i8 [[SUB]], [[Y:%.*]]
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[X]], [[Y]]
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP]], i8 127, i8 [[ADD]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %sub = sub i8 127, %x
+ %cmp = icmp sgt i8 %sub, %y
+ %add = add i8 %x, %y
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define <2 x i8> @sadd_sat_int_max_minus_x_vector(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @sadd_sat_int_max_minus_x_vector(
+; CHECK-NEXT: [[R:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
+; CHECK-NEXT: ret <2 x i8> [[R]]
+;
+ %sub = sub <2 x i8> <i8 127, i8 127>, %x
+ %cmp = icmp slt <2 x i8> %sub, %y
+ %add = add <2 x i8> %x, %y
+ %r = select <2 x i1> %cmp, <2 x i8> <i8 127, i8 127>, <2 x i8> %add
+ ret <2 x i8> %r
+}
+
+define i8 @sadd_sat_commuted_select(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_commuted_select(
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[X:%.*]], 127
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[Y:%.*]], 127
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP]], i8 [[ADD]], i8 127
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %cmp = icmp sge i8 %x, 127
+ %add = add i8 %x, %y
+ %r = select i1 %cmp, i8 %add, i8 127
+ ret i8 %r
+}
+
+define i8 @sadd_sat_commuted_add(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_commuted_add(
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[X:%.*]], 127
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[Y:%.*]], [[X]]
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP]], i8 127, i8 [[ADD]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %cmp = icmp sge i8 %x, 127
+ %add = add i8 %y, %x
+ %r = select i1 %cmp, i8 127, i8 %add
+ ret i8 %r
+}
+
+define i8 @sadd_sat_commuted_both(i8 %x, i8 %y) {
+; CHECK-LABEL: @sadd_sat_commuted_both(
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[X:%.*]], 127
+; CHECK-NEXT: [[ADD:%.*]] = add i8 [[Y:%.*]], 127
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP]], i8 [[ADD]], i8 127
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %cmp = icmp sge i8 %x, 127
+ %add = add i8 %y, %x
+ %r = select i1 %cmp, i8 %add, i8 127
+ ret i8 %r
+}
|
AZero13
force-pushed
the
sat-add
branch
2 times, most recently
from
638e6c8 to
9790621
Compare
AZero13
changed the title
AZero13
changed the title
AZero13
force-pushed
the
sat-add
branch
2 times, most recently
from
f844e3d to
afebcda
Compare
…umbers only https://alive2.llvm.org/ce/z/YGT5SN This is tricky because with positive numbers, we only go up, so we can in fact always hit the signed_max boundary. This is important because the intrinsic we use has the behavior of going the OTHER way, aka clamp to INT_MIN if it goes in that direction. And the range checking we do only works for positive numbers. Because of this issue, we can only do this for constants as well.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
https://alive2.llvm.org/ce/z/YGT5SN
https://alive2.llvm.org/ce/z/PVDxCw