Skip to content

Commit

Permalink
add check limit
Browse files Browse the repository at this point in the history
  • Loading branch information
@yujun777
yujun777 committed Nov 27, 2024
1 parent de8cfd1 commit 51d2208
Show file tree
Hide file tree
Showing 2 changed files with 176 additions and 52 deletions.
212 changes: 160 additions & 52 deletions ...ain/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,22 +38,23 @@
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal;
import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
import org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.types.DateTimeType;
import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.types.DateType;
import org.apache.doris.nereids.types.DateV2Type;
import org.apache.doris.nereids.types.DecimalV3Type;
import org.apache.doris.nereids.types.coercion.DateLikeType;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.TypeCoercionUtils;

import com.google.common.base.Preconditions;
Expand All @@ -64,7 +65,7 @@
import java.util.List;

/**
* simplify comparison
* simplify comparison, not support large int.
* such as: cast(c1 as DateV2) >= DateV2Literal --> c1 >= DateLiteral
* cast(c1 AS double) > 2.0 --> c1 >= 2 (c1 is integer like type)
*/
Expand Down Expand Up @@ -98,22 +99,24 @@ public static Expression simplify(ComparisonPredicate cp) {
Expression left = cp.left();
Expression right = cp.right();

// float like type: float, double
if (left.getDataType().isFloatLikeType() && right.getDataType().isFloatLikeType()) {
return processFloatLikeTypeCoercion(cp, left, right);
}
Expression result;

// decimalv3 type
if (left.getDataType() instanceof DecimalV3Type && right.getDataType() instanceof DecimalV3Type) {
return processDecimalV3TypeCoercion(cp, left, right);
// process type coercion
if (left.getDataType().isFloatLikeType() && right.getDataType().isFloatLikeType()) {
result = processFloatLikeTypeCoercion(cp, left, right);
} else if (left.getDataType() instanceof DecimalV3Type && right.getDataType() instanceof DecimalV3Type) {
result = processDecimalV3TypeCoercion(cp, left, right);
} else if (left.getDataType() instanceof DateLikeType && right.getDataType() instanceof DateLikeType) {
result = processDateLikeTypeCoercion(cp, left, right);
} else {
result = cp;
}

// date like type
if (left.getDataType() instanceof DateLikeType && right.getDataType() instanceof DateLikeType) {
return processDateLikeTypeCoercion(cp, left, right);
if (result instanceof ComparisonPredicate) {
result = processTypeRangeLimitComparision((ComparisonPredicate) result);
}

return cp;
return result;
}

private static Expression processComparisonPredicateDateTimeV2Literal(
Expand All @@ -128,17 +131,13 @@ private static Expression processComparisonPredicateDateTimeV2Literal(
if (right.getMicroSecond() == originValue) {
return comparisonPredicate.withChildren(left, right);
} else {
if (left.nullable()) {
// TODO: the ideal way is to return an If expr like:
// return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
// BooleanLiteral.of(false));
// but current fold constant rule can't handle such complex expr with null literal
// before supporting complex conjuncts with null literal folding rules,
// we use a trick way like this:
return new And(new IsNull(left), new NullLiteral(BooleanType.INSTANCE));
} else {
return BooleanLiteral.of(false);
}
// TODO: the ideal way is to return an If expr like:
// return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
// BooleanLiteral.of(false));
// but current fold constant rule can't handle such complex expr with null literal
// before supporting complex conjuncts with null literal folding rules,
// we use a trick way like this:
return ExpressionUtils.getFalse(left);
}
} else if (comparisonPredicate instanceof NullSafeEqual) {
long originValue = right.getMicroSecond();
Expand Down Expand Up @@ -239,18 +238,13 @@ private static Expression processDecimalV3TypeCoercion(ComparisonPredicate compa
comparisonPredicate.withChildren(left, new DecimalV3Literal(
literal.getValue().setScale(toScale, RoundingMode.UNNECESSARY))));
} catch (ArithmeticException e) {
if (left.nullable()) {
// TODO: the ideal way is to return an If expr like:
// return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
// BooleanLiteral.of(false));
// but current fold constant rule can't handle such complex expr with null literal
// before supporting complex conjuncts with null literal folding rules,
// we use a trick way like this:
return new And(new IsNull(left),
new NullLiteral(BooleanType.INSTANCE));
} else {
return BooleanLiteral.of(false);
}
// TODO: the ideal way is to return an If expr like:
// return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
// BooleanLiteral.of(false));
// but current fold constant rule can't handle such complex expr with null literal
// before supporting complex conjuncts with null literal folding rules,
// we use a trick way like this:
return ExpressionUtils.getFalse(left);
}
} else if (comparisonPredicate instanceof NullSafeEqual) {
try {
Expand Down Expand Up @@ -281,21 +275,18 @@ private static Expression processDecimalV3TypeCoercion(ComparisonPredicate compa
private static Expression processIntegerDecimalLiteralComparison(
ComparisonPredicate comparisonPredicate, Expression left, BigDecimal literal) {
// we only process isIntegerLikeType, which are tinyint, smallint, int, bigint
if (literal.compareTo(new BigDecimal(Long.MAX_VALUE)) <= 0) {
if (literal.compareTo(new BigDecimal(Long.MIN_VALUE)) >= 0
&& literal.compareTo(new BigDecimal(Long.MAX_VALUE)) <= 0) {
literal = literal.stripTrailingZeros();
if (literal.scale() > 0) {
if (comparisonPredicate instanceof EqualTo) {
if (left.nullable()) {
// TODO: the ideal way is to return an If expr like:
// return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
// BooleanLiteral.of(false));
// but current fold constant rule can't handle such complex expr with null literal
// before supporting complex conjuncts with null literal folding rules,
// we use a trick way like this:
return new And(new IsNull(left), new NullLiteral(BooleanType.INSTANCE));
} else {
return BooleanLiteral.of(false);
}
// TODO: the ideal way is to return an If expr like:
// return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE),
// BooleanLiteral.of(false));
// but current fold constant rule can't handle such complex expr with null literal
// before supporting complex conjuncts with null literal folding rules,
// we use a trick way like this:
return ExpressionUtils.getFalse(left);
} else if (comparisonPredicate instanceof NullSafeEqual) {
return BooleanLiteral.of(false);
} else if (comparisonPredicate instanceof GreaterThan
Expand All @@ -320,10 +311,127 @@ private static Expression processIntegerDecimalLiteralComparison(
return comparisonPredicate;
}

private static Expression processTypeRangeLimitComparision(ComparisonPredicate cp) {
Expression right = cp.right();
Expression left = cp.left();
if (left instanceof Cast) {
left = ((Cast) left).child();
}

BigDecimal literal = null;
if (right instanceof TinyIntLiteral) {
literal = new BigDecimal(((TinyIntLiteral) right).getValue());
} else if (right instanceof SmallIntLiteral) {
literal = new BigDecimal(((SmallIntLiteral) right).getValue());
} else if (right instanceof IntegerLiteral) {
literal = new BigDecimal(((IntegerLiteral) right).getValue());
} else if (right instanceof BigIntLiteral) {
literal = new BigDecimal(((BigIntLiteral) right).getValue());
} else if (right instanceof LargeIntLiteral) {
literal = new BigDecimal(((LargeIntLiteral) right).getValue());
} else if (right instanceof FloatLiteral) {
literal = new BigDecimal(((FloatLiteral) right).getValue());
} else if (right instanceof DoubleLiteral) {
literal = new BigDecimal(((DoubleLiteral) right).getValue());
} else if (right instanceof DecimalLiteral) {
literal = ((DecimalLiteral) right).getValue();
} else if (right instanceof DecimalV3Literal) {
literal = ((DecimalV3Literal) right).getValue();
}

if (literal == null) {
return cp;
}

BigDecimal minVal = null;
BigDecimal maxVal = null;
DataType leftType = left.getDataType();
if (leftType.isTinyIntType()) {
maxVal = new BigDecimal(Byte.MAX_VALUE);
minVal = new BigDecimal(Byte.MIN_VALUE);
} else if (leftType.isSmallIntType()) {
maxVal = new BigDecimal(Short.MAX_VALUE);
minVal = new BigDecimal(Short.MIN_VALUE);
} else if (leftType.isIntegerType()) {
maxVal = new BigDecimal(Integer.MAX_VALUE);
minVal = new BigDecimal(Integer.MIN_VALUE);
} else if (leftType.isBigIntType()) {
maxVal = new BigDecimal(Long.MAX_VALUE);
minVal = new BigDecimal(Long.MIN_VALUE);
} else if (leftType.isLargeIntType()) {
maxVal = new BigDecimal(LargeIntType.MAX_VALUE);
minVal = new BigDecimal(LargeIntType.MIN_VALUE);
} else if (leftType.isFloatType()) {
maxVal = new BigDecimal(Float.MAX_VALUE);
//minVal = BigDecimal.valueOf(-Float.MAX_VALUE);
minVal = BigDecimal.valueOf(Float.MIN_VALUE);
} else if (leftType.isDoubleType()) {
maxVal = new BigDecimal(Double.MAX_VALUE);
//minVal = BigDecimal.valueOf(-Double.MAX_VALUE);
minVal = BigDecimal.valueOf(Double.MIN_VALUE);
} else if (leftType.isDecimalV2Type()) {
// DecimalV2Type type = (DecimalV2Type) leftType;
} else if (leftType.isDecimalV3Type()) {
DecimalV3Type type = (DecimalV3Type) leftType;
}

if (minVal == null) {
return cp;
}

int cmpMin = literal.compareTo(minVal);
int cmpMax = literal.compareTo(maxVal);
if (cp instanceof EqualTo) {
if (cmpMin < 0 || cmpMax > 0) {
return ExpressionUtils.getFalse(left);
}
} else if (cp instanceof NullSafeEqual) {
if (cmpMin < 0 || cmpMax > 0) {
return BooleanLiteral.of(false);
}
} else if (cp instanceof GreaterThan) {
if (cmpMin < 0) {
return ExpressionUtils.getTrue(left);
}
if (cmpMax >= 0) {
return ExpressionUtils.getFalse(left);
}
} else if (cp instanceof GreaterThanEqual) {
if (cmpMin <= 0) {
return ExpressionUtils.getTrue(left);
}
if (cmpMax == 0) {
return new EqualTo(cp.left(), cp.right());
}
if (cmpMax > 0) {
return ExpressionUtils.getFalse(left);
}
} else if (cp instanceof LessThan) {
if (cmpMin <= 0) {
return ExpressionUtils.getFalse(left);
}
if (cmpMax > 0) {
return ExpressionUtils.getTrue(left);
}
} else if (cp instanceof LessThanEqual) {
if (cmpMin < 0) {
return ExpressionUtils.getFalse(left);
}
if (cmpMin == 0) {
return new EqualTo(cp.left(), cp.right());
}
if (cmpMax >= 0) {
return ExpressionUtils.getTrue(left);
}
}
return cp;
}

private static IntegerLikeLiteral convertDecimalToIntegerLikeLiteral(BigDecimal decimal) {
Preconditions.checkArgument(
decimal.scale() <= 0 && decimal.compareTo(new BigDecimal(Long.MAX_VALUE)) <= 0,
"decimal literal must have 0 scale and smaller than Long.MAX_VALUE");
Preconditions.checkArgument(decimal.scale() <= 0
&& decimal.compareTo(new BigDecimal(Long.MIN_VALUE)) >= 0
&& decimal.compareTo(new BigDecimal(Long.MAX_VALUE)) <= 0,
"decimal literal must have 0 scale and in range [Long.MIN_VALUE, Long.MAX_VALUE]");
long val = decimal.longValue();
if (val >= Byte.MIN_VALUE && val <= Byte.MAX_VALUE) {
return new TinyIntLiteral((byte) val);
Expand Down
16 changes: 16 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,22 @@ public static Optional<Expression> optionalAnd(List<Expression> expressions) {
}
}

public static Expression getFalse(Expression expression) {
if (expression.nullable()) {
return new And(new IsNull(expression), new NullLiteral(BooleanType.INSTANCE));
} else {
return BooleanLiteral.FALSE;
}
}

public static Expression getTrue(Expression expression) {
if (expression.nullable()) {
return new Or(new Not(new IsNull(expression)), new NullLiteral(BooleanType.INSTANCE));
} else {
return BooleanLiteral.TRUE;
}
}

/**
* And two list.
*/
Expand Down

0 comments on commit 51d2208

Please sign in to comment.