From 723b774f14d7f77bf6feedfe0d3660495cb0a590 Mon Sep 17 00:00:00 2001 From: "pavel.voropaev" Date: Tue, 18 Apr 2023 10:32:36 +0100 Subject: [PATCH 1/4] Use 'Numeric' instead of 'Float' as BQ column type when field type in the json schema is number (close #112) --- .../iglu.schemaddl/bigquery/Field.scala | 8 +- .../iglu.schemaddl/bigquery/Row.scala | 2 +- .../iglu.schemaddl/bigquery/Suggestion.scala | 72 ++++++++- .../iglu.schemaddl/bigquery/Type.scala | 18 ++- .../jsonschema/suggestion/baseTypes.scala | 28 ++++ .../jsonschema/suggestion/decimals.scala | 71 +++++++++ .../iglu.schemaddl/parquet/Field.scala | 24 ++- .../iglu.schemaddl/parquet/Suggestion.scala | 68 +-------- .../iglu.schemaddl/parquet/Type.scala | 43 ++++-- .../iglu/schemaddl/bigquery/FieldSpec.scala | 138 +++++++++++++++++- .../iglu/schemaddl/bigquery/RowSpec.scala | 2 +- 11 files changed, 383 insertions(+), 91 deletions(-) create mode 100644 modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/baseTypes.scala create mode 100644 modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/decimals.scala diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Field.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Field.scala index 65076d8b..45298213 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Field.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Field.scala @@ -42,27 +42,27 @@ object Field { case Some(types) if types.possiblyWithNull(CommonProperties.Type.Object) => val subfields = topSchema.properties.map(_.value).getOrElse(Map.empty) if (subfields.isEmpty) { - Suggestion.finalSuggestion(topSchema, required)(name) + Suggestion.finalSuggestion(topSchema, required || types.nullable)(name) } else { val requiredKeys = topSchema.required.toList.flatMap(_.value) val fields = subfields.map { case (key, schema) => build(key, schema, requiredKeys.contains(key)) } val subFields = fields.toList.sortBy(field => (Mode.sort(field.mode), field.name)) - Field(name, Type.Record(subFields), Mode.required(required)) + Field(name, Type.Record(subFields), Mode.required(required || types.nullable)) } case Some(types) if types.possiblyWithNull(CommonProperties.Type.Array) => topSchema.items match { case Some(ArrayProperty.Items.ListItems(schema)) => build(name, schema, false).copy(mode = Mode.Repeated) case _ => - Suggestion.finalSuggestion(topSchema, required)(name) + Suggestion.finalSuggestion(topSchema, required || types.nullable)(name) } case _ => Suggestion.suggestions .find(suggestion => suggestion(topSchema, required).isDefined) .flatMap(_.apply(topSchema, required)) - .getOrElse(Suggestion.finalSuggestion(topSchema, required)) + .getOrElse(Suggestion.finalSuggestion(topSchema, required = false)) .apply(name) } } diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Row.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Row.scala index 0b751434..f0bbc4cb 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Row.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Row.scala @@ -59,7 +59,7 @@ object Row { value.asNumber.flatMap(_.toLong).fold(WrongType(value, fieldType).invalidNel[Row])(Primitive(_).validNel) case Type.Float => value.asNumber.flatMap(_.toBigDecimal.map(_.bigDecimal)).fold(WrongType(value, fieldType).invalidNel[Row])(Primitive(_).validNel) - case Type.Numeric => + case Type.Numeric(_, _) => value.asNumber.flatMap(_.toBigDecimal.map(_.bigDecimal)).fold(WrongType(value, fieldType).invalidNel[Row])(Primitive(_).validNel) case Type.Timestamp => value.asString.fold(WrongType(value, fieldType).invalidNel[Row])(Primitive(_).validNel) diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala index 8a90141f..5708777f 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala @@ -13,9 +13,10 @@ package com.snowplowanalytics.iglu.schemaddl.bigquery import io.circe._ - import com.snowplowanalytics.iglu.schemaddl.jsonschema.Schema -import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.{CommonProperties, StringProperty} +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.decimals +import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.{CommonProperties, StringProperty, NumberProperty} +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.baseTypes.NullableWrapper object Suggestion { @@ -24,7 +25,7 @@ object Suggestion { case Some(CommonProperties.Type.String) => Some(name => Field(name, Type.String, Mode.required(required))) case Some(types) if types.nullable(CommonProperties.Type.String) => - Some(name => Field(name, Type.String, Mode.Nullable) ) + Some(name => Field(name, Type.String, Mode.Nullable)) case _ => None } @@ -40,19 +41,55 @@ object Suggestion { val integerSuggestion: Suggestion = (schema, required) => schema.`type` match { case Some(CommonProperties.Type.Integer) => - Some(name => Field(name, Type.Integer, Mode.required(required))) + Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.required(required))) case Some(CommonProperties.Type.Union(types)) if withNull(types, CommonProperties.Type.Integer) => - Some(name => Field(name, Type.Integer, Mode.Nullable)) + Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.Nullable)) case _ => None } + val numericSuggestion: Suggestion = (schema, required) => schema.`type` match { + case Some(CommonProperties.Type.Number) => + schema.multipleOf match { + case Some(NumberProperty.MultipleOf.IntegerMultipleOf(_)) => + Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.required(required))) + case Some(mult: NumberProperty.MultipleOf.NumberMultipleOf) => + Some(name => Field(name, numericWithMultiple(mult, schema.maximum, schema.minimum), Mode.required(required))) + case None => None + } + case Some(CommonProperties.Type.Union(types)) if withNull(types, CommonProperties.Type.Number) => + schema.multipleOf match { + case Some(NumberProperty.MultipleOf.IntegerMultipleOf(_)) => + Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.Nullable)) + case Some(mult: NumberProperty.MultipleOf.NumberMultipleOf) => + Some(name => Field(name, numericWithMultiple(mult, schema.maximum, schema.minimum), Mode.Nullable)) + case None => None + } + case Some(CommonProperties.Type.Union(types)) if onlyNumeric(types, true) => + schema.multipleOf match { + case Some(NumberProperty.MultipleOf.IntegerMultipleOf(_)) => + Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.Nullable)) + case Some(mult: NumberProperty.MultipleOf.NumberMultipleOf) => + Some(name => Field(name, numericWithMultiple(mult, schema.maximum, schema.minimum), Mode.Nullable)) + case None => None + } + case Some(CommonProperties.Type.Union(types)) if onlyNumeric(types, false) => + schema.multipleOf match { + case Some(NumberProperty.MultipleOf.IntegerMultipleOf(_)) => + Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.required(required))) + case Some(mult: NumberProperty.MultipleOf.NumberMultipleOf) => + Some(name => Field(name, numericWithMultiple(mult, schema.maximum, schema.minimum), Mode.required(required))) + case None => None + } + case _ => None + } + val floatSuggestion: Suggestion = (schema, required) => schema.`type` match { case Some(CommonProperties.Type.Number) => Some(name => Field(name, Type.Float, Mode.required(required))) case Some(CommonProperties.Type.Union(types)) if onlyNumeric(types, true) => Some(name => Field(name, Type.Float, Mode.Nullable)) - case Some(CommonProperties.Type.Union(types)) if onlyNumeric(types, false) => + case Some(CommonProperties.Type.Union(types)) if onlyNumeric(types, false) => Some(name => Field(name, Type.Float, Mode.required(required))) case Some(CommonProperties.Type.Union(types)) if withNull(types, CommonProperties.Type.Number) => Some(name => Field(name, Type.Float, Mode.Nullable)) @@ -66,6 +103,15 @@ object Suggestion { case _ => None } + private def numericWithMultiple(mult: NumberProperty.MultipleOf.NumberMultipleOf, + maximum: Option[NumberProperty.Maximum], + minimum: Option[NumberProperty.Minimum]): Type = + Type.fromGenericType(decimals.numericWithMultiple(mult, maximum, minimum)) + + + // (Field.JsonNullability.fromNullableWrapper) + + // `date-time` format usually means zoned format, which corresponds to BQ Timestamp val timestampSuggestion: Suggestion = (schema, required) => (schema.`type`, schema.format) match { @@ -95,14 +141,18 @@ object Suggestion { booleanSuggestion, stringSuggestion, integerSuggestion, + numericSuggestion, floatSuggestion, complexEnumSuggestion ) private[iglu] def fromEnum(enums: List[Json], required: Boolean): String => Field = { def isString(json: Json) = json.isString || json.isNull + def isInteger(json: Json) = json.asNumber.exists(_.toBigInt.isDefined) || json.isNull + def isNumeric(json: Json) = json.isNumber || json.isNull + val noNull: Boolean = !enums.contains(Json.Null) if (enums.forall(isString)) { @@ -110,7 +160,15 @@ object Suggestion { } else if (enums.forall(isInteger)) { name => Field(name, Type.Integer, Mode.required(required && noNull)) } else if (enums.forall(isNumeric)) { - name => Field(name, Type.Float, Mode.required(required && noNull)) + name => + decimals.numericEnum(enums).map { + case NullableWrapper.NullableValue(t) => Field(name, Type.fromGenericType(t), Mode.required(required && noNull)) + case NullableWrapper.NotNullValue(t) => Field(name, Type.fromGenericType(t), Mode.required(required && noNull)) + } match { + case Some(value) => value + // Unreachable as `None` here would mean that some `enums.forall(isNumeric)` did not work. + case None => Field(name, Type.Float, Mode.required(required && noNull)) + } } else { name => Field(name, Type.String, Mode.required(required && noNull)) } diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Type.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Type.scala index 8fc8c3fd..56f223fd 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Type.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Type.scala @@ -12,18 +12,34 @@ */ package com.snowplowanalytics.iglu.schemaddl.bigquery +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.baseTypes._ /** BigQuery field type; "array" and "null" are expressed via `Mode` */ sealed trait Type extends Product with Serializable object Type { case object String extends Type + case object Boolean extends Type + case object Integer extends Type + case object Float extends Type - case object Numeric extends Type + + case class Numeric(precision: Int, scale: Int) extends Type + case object Date extends Type + case object DateTime extends Type + case object Timestamp extends Type + case class Record(fields: List[Field]) extends Type + + def fromGenericType(`type`: BaseType) = `type` match { + case BaseType.Double => Float + case BaseType.Int32 => Integer + case BaseType.Int64 => Integer + case BaseType.Decimal(precision, scale) => Numeric(precision, scale) + } } diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/baseTypes.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/baseTypes.scala new file mode 100644 index 00000000..66035c47 --- /dev/null +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/baseTypes.scala @@ -0,0 +1,28 @@ +package com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion + +object baseTypes { + sealed trait BaseType extends Product with Serializable + + object BaseType { + case object Double extends BaseType + + case object Int32 extends BaseType + + case object Int64 extends BaseType + + case class Decimal(precision: Int, scale: Int) extends BaseType + } + + sealed trait NullableWrapper + + object NullableWrapper { + + case class NullableValue(`type`: BaseType) extends NullableWrapper + + case class NotNullValue(`type`: BaseType) extends NullableWrapper + + def fromBool(`type`: BaseType, nullable: Boolean): NullableWrapper = + if (nullable) NullableWrapper.NullableValue(`type`) + else NullableWrapper.NotNullValue(`type`) + } +} diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/decimals.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/decimals.scala new file mode 100644 index 00000000..69e9bc9e --- /dev/null +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/decimals.scala @@ -0,0 +1,71 @@ +package com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion + +import baseTypes._ +import io.circe.Json +import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.NumberProperty +import com.snowplowanalytics.iglu.schemaddl.jsonschema.Schema + +object decimals { + + def integerType(schema: Schema): BaseType = + (schema.minimum, schema.maximum) match { + case (Some(min), Some(max)) => + val minDecimal = min.getAsDecimal + val maxDecimal = max.getAsDecimal + if (maxDecimal <= Int.MaxValue && minDecimal >= Int.MinValue) BaseType.Int32 + else if (maxDecimal <= Long.MaxValue && minDecimal >= Long.MinValue) BaseType.Int64 + else BaseType.Decimal( + (maxDecimal.precision - maxDecimal.scale).max(minDecimal.precision - minDecimal.scale), 0 + ) + case _ => BaseType.Int64 + } + + def numericWithMultiple(mult: NumberProperty.MultipleOf.NumberMultipleOf, + maximum: Option[NumberProperty.Maximum], + minimum: Option[NumberProperty.Minimum]): BaseType = + (maximum, minimum) match { + case (Some(max), Some(min)) => + val topPrecision = max match { + case NumberProperty.Maximum.IntegerMaximum(max) => + BigDecimal(max).precision + mult.value.scale + case NumberProperty.Maximum.NumberMaximum(max) => + max.precision - max.scale + mult.value.scale + } + val bottomPrecision = min match { + case NumberProperty.Minimum.IntegerMinimum(min) => + BigDecimal(min).precision + mult.value.scale + case NumberProperty.Minimum.NumberMinimum(min) => + min.precision - min.scale + mult.value.scale + } + + BaseType.Decimal(topPrecision.max(bottomPrecision), mult.value.scale) + + case _ => + BaseType.Double + } + + + def numericEnum(enums: List[Json]): Option[NullableWrapper] = { + def go(scale: Int, max: BigDecimal, nullable: Boolean, enums: List[Json]): Option[NullableWrapper] = + enums match { + case Nil => + val t = if ((scale == 0) && (max <= Int.MaxValue)) BaseType.Int32 + else if ((scale == 0) && (max <= Long.MaxValue)) BaseType.Int64 + else BaseType.Decimal(max.precision - max.scale + scale, scale) + + Some(NullableWrapper.fromBool(t, nullable)) + case Json.Null :: tail => go(scale, max, true, tail) + case h :: tail => + h.asNumber.flatMap(_.toBigDecimal) match { + case Some(bigDecimal) => + val nextScale = scale.max(bigDecimal.scale) + val nextMax = (if (bigDecimal > 0) bigDecimal else -bigDecimal).max(max) + go(nextScale, nextMax, nullable, tail) + case None => None + } + } + + go(0, 0, false, enums) + } + +} diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Field.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Field.scala index 614c4cda..98e77c27 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Field.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Field.scala @@ -12,13 +12,15 @@ */ package com.snowplowanalytics.iglu.schemaddl.parquet +import Type._ import com.snowplowanalytics.iglu.schemaddl.StringUtils import com.snowplowanalytics.iglu.schemaddl.jsonschema.Schema import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.{ArrayProperty, CommonProperties} import com.snowplowanalytics.iglu.schemaddl.jsonschema.mutate.Mutate +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.baseTypes.NullableWrapper -case class Field(name: String, - fieldType: Type, +case class Field(name: String, + fieldType: Type, nullability: Type.Nullability) object Field { @@ -67,8 +69,20 @@ object Field { private[parquet] object JsonNullability { case object ExplicitlyNullable extends JsonNullability + case object NoExplicitNull extends JsonNullability + def fromNullableWrapper(wrapper: NullableWrapper): NullableType = wrapper match { + case NullableWrapper.NullableValue(t) => NullableType( + value = fromGenericType(t), + nullability = JsonNullability.ExplicitlyNullable + ) + case NullableWrapper.NotNullValue(t) => NullableType( + value = fromGenericType(t), + nullability = JsonNullability.NoExplicitNull + ) + } + def extractFrom(`type`: CommonProperties.Type): JsonNullability = { if (`type`.nullable) { JsonNullability.ExplicitlyNullable @@ -81,13 +95,13 @@ object Field { topSchema.`type` match { case Some(types) if types.possiblyWithNull(CommonProperties.Type.Object) => NullableType( - value = buildObjectType(topSchema), + value = buildObjectType(topSchema), nullability = JsonNullability.extractFrom(types) ) case Some(types) if types.possiblyWithNull(CommonProperties.Type.Array) => - NullableType( - value = buildArrayType(topSchema), + NullableType( + value = buildArrayType(topSchema), nullability = JsonNullability.extractFrom(types) ) diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Suggestion.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Suggestion.scala index eab58ffa..1bd199cd 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Suggestion.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Suggestion.scala @@ -12,8 +12,8 @@ */ package com.snowplowanalytics.iglu.schemaddl.parquet -import cats.implicits._ import com.snowplowanalytics.iglu.schemaddl.jsonschema.Schema +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.decimals import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.{CommonProperties, NumberProperty, StringProperty} import io.circe._ @@ -80,55 +80,9 @@ private[parquet] object Suggestion { private def numericWithMultiple(mult: NumberProperty.MultipleOf.NumberMultipleOf, maximum: Option[NumberProperty.Maximum], minimum: Option[NumberProperty.Minimum]): Type = - (maximum, minimum) match { - case (Some(max), Some(min)) => - val topPrecision = max match { - case NumberProperty.Maximum.IntegerMaximum(max) => - BigDecimal(max).precision + mult.value.scale - case NumberProperty.Maximum.NumberMaximum(max) => - max.precision - max.scale + mult.value.scale - } - val bottomPrecision = min match { - case NumberProperty.Minimum.IntegerMinimum(min) => - BigDecimal(min).precision + mult.value.scale - case NumberProperty.Minimum.NumberMinimum(min) => - min.precision - min.scale + mult.value.scale - } - Type.DecimalPrecision.of(topPrecision.max(bottomPrecision)) match { - case Some(precision) => - Type.Decimal(precision, mult.value.scale) - case None => - Type.Double - } - case _ => - Type.Double - } + Type.fromGenericType(decimals.numericWithMultiple(mult, maximum, minimum)) - - private def numericEnum(enums: List[Json]): Option[Field.NullableType] = { - def go(scale: Int, max: BigDecimal, nullable: Field.JsonNullability, enums: List[Json]): Option[Field.NullableType] = - enums match { - case Nil => - val t = if (scale === 0 && max <= Int.MaxValue) Type.Integer - else if (scale === 0 && max <= Long.MaxValue) Type.Long - else { - val precision = (max.precision - max.scale) + scale - Type.DecimalPrecision.of(precision).fold[Type](Type.Double)(Type.Decimal(_, scale)) - } - Some(Field.NullableType(t, nullable)) - case Json.Null :: tail => go(scale, max, Field.JsonNullability.ExplicitlyNullable, tail) - case h :: tail => - h.asNumber.flatMap(_.toBigDecimal) match { - case Some(bigDecimal) => - val nextScale = scale.max(bigDecimal.scale) - val nextMax = (if (bigDecimal > 0) bigDecimal else -bigDecimal).max(max) - go(nextScale, nextMax, nullable, tail) - case None => None - } - } - - go(0, 0, Field.JsonNullability.NoExplicitNull, enums) - } + private def numericEnum(enums: List[Json]): Option[Field.NullableType] = decimals.numericEnum(enums).map(Field.JsonNullability.fromNullableWrapper) private def stringEnum(enums: List[Json]): Option[Field.NullableType] = { def go(nullable: Field.JsonNullability, enums: List[Json]): Option[Field.NullableType] = @@ -138,6 +92,7 @@ private[parquet] object Suggestion { case h :: tail if h.isString => go(nullable, tail) case _ => None } + go(Field.JsonNullability.NoExplicitNull, enums) } @@ -146,19 +101,8 @@ private[parquet] object Suggestion { Field.NullableType(Type.Json, nullable) } - private def integerType(schema: Schema): Type = - (schema.minimum, schema.maximum) match { - case (Some(min), Some(max)) => - val minDecimal = min.getAsDecimal - val maxDecimal = max.getAsDecimal - if (maxDecimal <= Int.MaxValue && minDecimal >= Int.MinValue) Type.Integer - else if (maxDecimal <= Long.MaxValue && minDecimal >= Long.MinValue) Type.Long - else Type.DecimalPrecision - .of((maxDecimal.precision - maxDecimal.scale).max(minDecimal.precision - minDecimal.scale)) - .fold[Type](Type.Double)(Type.Decimal(_, 0)) - case _ => Type.Long - } - + private def integerType(schema: Schema): Type = Type.fromGenericType(decimals.integerType(schema)) + private def onlyNumeric(types: CommonProperties.Type): Boolean = types match { case CommonProperties.Type.Number => true diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Type.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Type.scala index 0ecd94fa..f296d773 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Type.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Type.scala @@ -13,20 +13,30 @@ package com.snowplowanalytics.iglu.schemaddl.parquet import cats.Eq +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.baseTypes._ sealed trait Type extends Product with Serializable object Type { - case object String extends Type - case object Boolean extends Type - case object Integer extends Type - case object Long extends Type - case object Double extends Type - case class Decimal(precision: DecimalPrecision, scale: Int) extends Type - case object Date extends Type - case object Timestamp extends Type - case class Struct(fields: List[Field]) extends Type + case object String extends Type + + case object Boolean extends Type + + case object Integer extends Type + + case object Long extends Type + + case object Double extends Type + + case class Decimal(precision: DecimalPrecision, scale: Int) extends Type + + case object Date extends Type + + case object Timestamp extends Type + + case class Struct(fields: List[Field]) extends Type + case class Array(element: Type, nullability: Nullability) extends Type /* Fallback type for when json schema does not map to a parquet primitive type (e.g. unions) @@ -41,6 +51,7 @@ object Type { sealed trait Nullability { def nullable: Boolean + def required: Boolean = !nullable } @@ -48,15 +59,19 @@ object Type { case object Nullable extends Nullability { override def nullable: Boolean = true } + case object Required extends Nullability { override def nullable: Boolean = false } } sealed trait DecimalPrecision + object DecimalPrecision { case object Digits9 extends DecimalPrecision // Int32 physical type + case object Digits18 extends DecimalPrecision // Int64 physical type + case object Digits38 extends DecimalPrecision // Fixed length byte array physical type. def of(precision: Int): Option[DecimalPrecision] = @@ -71,4 +86,14 @@ object Type { case Digits38 => 38 } } + + def fromGenericType(`type`: BaseType) = `type` match { + case BaseType.Double => Double + case BaseType.Int32 => Integer + case BaseType.Int64 => Long + case BaseType.Decimal(precision, scale) => DecimalPrecision.of(precision) match { + case Some(value) => Decimal(value, scale) + case None => Double + } + } } diff --git a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/FieldSpec.scala b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/FieldSpec.scala index 39a2795b..5bb6bf1a 100644 --- a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/FieldSpec.scala +++ b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/FieldSpec.scala @@ -23,6 +23,11 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" build generates repeated string for empty schema in items $e7 build generates repeated record for nullable array $e8 normalName handles camel case and disallowed characters $e9 + build generates nullable field for oneOf types $e10 + build generates nullable field for nullable object without nested keys $e11 + build generates nullable field for nullable array without items $e12 + build generates numeric/decimal for enums $e13 + build generates numeric/decimal for multipleof $e14 """ def e1 = { @@ -91,7 +96,7 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" ) Field.build("foo", input, false) must beEqualTo(expected) - } + } def e3 = { val input = SpecHelpers.parseSchema( @@ -231,5 +236,136 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" (fieldNormalName("1test1,Test2Test3Test4.test5;test6") must beEqualTo("_1test1_test2_test3_test4_test5_test6")) } + def e10 = { + val input = SpecHelpers.parseSchema( + """ + | { + | "type": "object", + | "required": ["xyz"], + | "properties": { + | "xyz": { + | "oneOf": [ + | {"type": "string"}, + | {"type": "number"}, + | {"type": "null"} + | ] + | } + | } + | } + """.stripMargin) + + val expected = Field( + "foo", + Type.Record(List( + Field("xyz", Type.String, Mode.Nullable) + )), + Mode.Nullable + ) + + Field.build("foo", input, false) must beEqualTo(expected) + } + + def e11 = { + val input = SpecHelpers.parseSchema( + """ + | { + | "type": "object", + | "required": ["xyz"], + | "properties": { + | "xyz": { + | "type": ["object", "null"] + | } + | } + | } + """.stripMargin) + + val expected = Field( + "foo", + Type.Record(List( + Field("xyz", Type.String, Mode.Nullable) + )), + Mode.Nullable + ) + + Field.build("foo", input, false) must beEqualTo(expected) + } + + def e12 = { + val input = SpecHelpers.parseSchema( + """ + | { + | "type": "object", + | "required": ["xyz"], + | "properties": { + | "xyz": { + | "type": ["array", "null"] + | } + | } + | } + """.stripMargin) + + val expected = Field( + "foo", + Type.Record(List( + Field("xyz", Type.String, Mode.Nullable) + )), + Mode.Nullable + ) + + Field.build("foo", input, false) must beEqualTo(expected) + } + + def e13 = { + val input = SpecHelpers.parseSchema( + """ + | { + | "type": "object", + | "required": ["xyz"], + | "properties": { + | "xyz": { + | "enum": [10, 1.12, 1e9] + | } + | } + | } + """.stripMargin) + + val expected = Field( + "foo", + Type.Record(List( + Field("xyz", Type.Numeric(12,2), Mode.Required) + )), + Mode.Nullable + ) + + Field.build("foo", input, false) must beEqualTo(expected) + } + + def e14 = { + val input = SpecHelpers.parseSchema( + """ + | { + | "type": "object", + | "required": ["xyz"], + | "properties": { + | "xyz": { + | "type": ["number", "null"], + | "multipleOf": 0.001, + | "maximum": 2, + | "minimum": 1 + | } + | } + | } + """.stripMargin) + + val expected = Field( + "foo", + Type.Record(List( + Field("xyz", Type.Numeric(4,3), Mode.Nullable) + )), + Mode.Nullable + ) + + Field.build("foo", input, false) must beEqualTo(expected) + } private def fieldNormalName(name: String) = Field(name, Type.String, Mode.Nullable).normalName } diff --git a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/RowSpec.scala b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/RowSpec.scala index c9a99471..73d7ee2a 100644 --- a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/RowSpec.scala +++ b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/RowSpec.scala @@ -36,7 +36,7 @@ class RowSpec extends org.specs2.Specification { def is = s2""" def e1 = { val string = castValue(Type.String)(json""""foo"""") must beValid(Primitive("foo")) val int = castValue(Type.Integer)(json"-43") must beValid(Primitive(-43)) - val num = castValue(Type.Numeric)(json"-87.98") must beValid(Primitive(new java.math.BigDecimal("-87.98"))) + val num = castValue(Type.Numeric(1,0))(json"-87.98") must beValid(Primitive(new java.math.BigDecimal("-87.98"))) val bool = castValue(Type.Boolean)(Json.fromBoolean(false)) must beValid(Primitive(false)) string and int and num and bool } From 75d61254bb47e755ccd0ccd2b6edb2023542b418 Mon Sep 17 00:00:00 2001 From: "pavel.voropaev" Date: Wed, 26 Apr 2023 19:09:10 +0100 Subject: [PATCH 2/4] review feedback --- .../iglu.schemaddl/bigquery/Suggestion.scala | 2 +- .../iglu.schemaddl/bigquery/Type.scala | 12 ++++---- .../jsonschema/suggestion/baseTypes.scala | 28 ----------------- .../jsonschema/suggestion/decimals.scala | 30 ++++++++++--------- .../jsonschema/suggestion/numericType.scala | 25 ++++++++++++++++ .../iglu.schemaddl/parquet/Field.scala | 2 +- .../iglu.schemaddl/parquet/Type.scala | 12 ++++---- .../iglu/schemaddl/bigquery/FieldSpec.scala | 18 +++++------ 8 files changed, 64 insertions(+), 65 deletions(-) delete mode 100644 modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/baseTypes.scala create mode 100644 modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/numericType.scala diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala index 5708777f..fab73a58 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala @@ -16,7 +16,7 @@ import io.circe._ import com.snowplowanalytics.iglu.schemaddl.jsonschema.Schema import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.decimals import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.{CommonProperties, StringProperty, NumberProperty} -import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.baseTypes.NullableWrapper +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.numericType.NullableWrapper object Suggestion { diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Type.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Type.scala index 56f223fd..f4ddaa00 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Type.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Type.scala @@ -12,7 +12,7 @@ */ package com.snowplowanalytics.iglu.schemaddl.bigquery -import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.baseTypes._ +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.numericType._ /** BigQuery field type; "array" and "null" are expressed via `Mode` */ sealed trait Type extends Product with Serializable @@ -35,11 +35,11 @@ object Type { case class Record(fields: List[Field]) extends Type - def fromGenericType(`type`: BaseType) = `type` match { - case BaseType.Double => Float - case BaseType.Int32 => Integer - case BaseType.Int64 => Integer - case BaseType.Decimal(precision, scale) => Numeric(precision, scale) + def fromGenericType(`type`: NumericType) = `type` match { + case NumericType.Double => Float + case NumericType.Int32 => Integer + case NumericType.Int64 => Integer + case NumericType.Decimal(precision, scale) => Numeric(precision, scale) } } diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/baseTypes.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/baseTypes.scala deleted file mode 100644 index 66035c47..00000000 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/baseTypes.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion - -object baseTypes { - sealed trait BaseType extends Product with Serializable - - object BaseType { - case object Double extends BaseType - - case object Int32 extends BaseType - - case object Int64 extends BaseType - - case class Decimal(precision: Int, scale: Int) extends BaseType - } - - sealed trait NullableWrapper - - object NullableWrapper { - - case class NullableValue(`type`: BaseType) extends NullableWrapper - - case class NotNullValue(`type`: BaseType) extends NullableWrapper - - def fromBool(`type`: BaseType, nullable: Boolean): NullableWrapper = - if (nullable) NullableWrapper.NullableValue(`type`) - else NullableWrapper.NotNullValue(`type`) - } -} diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/decimals.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/decimals.scala index 69e9bc9e..cf2edb16 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/decimals.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/decimals.scala @@ -1,28 +1,28 @@ package com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion -import baseTypes._ +import numericType._ import io.circe.Json import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.NumberProperty import com.snowplowanalytics.iglu.schemaddl.jsonschema.Schema -object decimals { +private[schemaddl] object decimals { - def integerType(schema: Schema): BaseType = + def integerType(schema: Schema): NumericType = (schema.minimum, schema.maximum) match { case (Some(min), Some(max)) => val minDecimal = min.getAsDecimal val maxDecimal = max.getAsDecimal - if (maxDecimal <= Int.MaxValue && minDecimal >= Int.MinValue) BaseType.Int32 - else if (maxDecimal <= Long.MaxValue && minDecimal >= Long.MinValue) BaseType.Int64 - else BaseType.Decimal( + if (maxDecimal <= Int.MaxValue && minDecimal >= Int.MinValue) NumericType.Int32 + else if (maxDecimal <= Long.MaxValue && minDecimal >= Long.MinValue) NumericType.Int64 + else NumericType.Decimal( (maxDecimal.precision - maxDecimal.scale).max(minDecimal.precision - minDecimal.scale), 0 ) - case _ => BaseType.Int64 + case _ => NumericType.Int64 } def numericWithMultiple(mult: NumberProperty.MultipleOf.NumberMultipleOf, maximum: Option[NumberProperty.Maximum], - minimum: Option[NumberProperty.Minimum]): BaseType = + minimum: Option[NumberProperty.Minimum]): NumericType = (maximum, minimum) match { case (Some(max), Some(min)) => val topPrecision = max match { @@ -38,10 +38,10 @@ object decimals { min.precision - min.scale + mult.value.scale } - BaseType.Decimal(topPrecision.max(bottomPrecision), mult.value.scale) + NumericType.Decimal(topPrecision.max(bottomPrecision), mult.value.scale) case _ => - BaseType.Double + NumericType.Double } @@ -49,11 +49,13 @@ object decimals { def go(scale: Int, max: BigDecimal, nullable: Boolean, enums: List[Json]): Option[NullableWrapper] = enums match { case Nil => - val t = if ((scale == 0) && (max <= Int.MaxValue)) BaseType.Int32 - else if ((scale == 0) && (max <= Long.MaxValue)) BaseType.Int64 - else BaseType.Decimal(max.precision - max.scale + scale, scale) + val t = if ((scale == 0) && (max <= Int.MaxValue)) NumericType.Int32 + else if ((scale == 0) && (max <= Long.MaxValue)) NumericType.Int64 + else NumericType.Decimal(max.precision - max.scale + scale, scale) + + Some(if (nullable) NullableWrapper.NullableValue(t) + else NullableWrapper.NotNullValue(t)) - Some(NullableWrapper.fromBool(t, nullable)) case Json.Null :: tail => go(scale, max, true, tail) case h :: tail => h.asNumber.flatMap(_.toBigDecimal) match { diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/numericType.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/numericType.scala new file mode 100644 index 00000000..d2496a33 --- /dev/null +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/numericType.scala @@ -0,0 +1,25 @@ +package com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion + +private[schemaddl] object numericType { + sealed trait NumericType extends Product with Serializable + + object NumericType { + case object Double extends NumericType + + case object Int32 extends NumericType + + case object Int64 extends NumericType + + case class Decimal(precision: Int, scale: Int) extends NumericType + } + + sealed trait NullableWrapper + + object NullableWrapper { + + case class NullableValue(value: NumericType) extends NullableWrapper + + case class NotNullValue(value: NumericType) extends NullableWrapper + + } +} diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Field.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Field.scala index 98e77c27..044152fb 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Field.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Field.scala @@ -17,7 +17,7 @@ import com.snowplowanalytics.iglu.schemaddl.StringUtils import com.snowplowanalytics.iglu.schemaddl.jsonschema.Schema import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.{ArrayProperty, CommonProperties} import com.snowplowanalytics.iglu.schemaddl.jsonschema.mutate.Mutate -import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.baseTypes.NullableWrapper +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.numericType.NullableWrapper case class Field(name: String, fieldType: Type, diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Type.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Type.scala index f296d773..1d00bc57 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Type.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Type.scala @@ -13,7 +13,7 @@ package com.snowplowanalytics.iglu.schemaddl.parquet import cats.Eq -import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.baseTypes._ +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.numericType._ sealed trait Type extends Product with Serializable @@ -87,11 +87,11 @@ object Type { } } - def fromGenericType(`type`: BaseType) = `type` match { - case BaseType.Double => Double - case BaseType.Int32 => Integer - case BaseType.Int64 => Long - case BaseType.Decimal(precision, scale) => DecimalPrecision.of(precision) match { + def fromGenericType(`type`: NumericType) = `type` match { + case NumericType.Double => Double + case NumericType.Int32 => Integer + case NumericType.Int64 => Long + case NumericType.Decimal(precision, scale) => DecimalPrecision.of(precision) match { case Some(value) => Decimal(value, scale) case None => Double } diff --git a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/FieldSpec.scala b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/FieldSpec.scala index 5bb6bf1a..1c22df1d 100644 --- a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/FieldSpec.scala +++ b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/FieldSpec.scala @@ -63,7 +63,7 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" )), Mode.Nullable ), - Field("stringKey", Type.String,Mode.Nullable))), + Field("stringKey", Type.String, Mode.Nullable))), Mode.Nullable ) @@ -161,9 +161,9 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" |} """.stripMargin) - val expected = Field("foo",Type.Record(List( - Field("union",Type.String,Mode.Nullable) - )),Mode.Nullable) + val expected = Field("foo", Type.Record(List( + Field("union", Type.String, Mode.Nullable) + )), Mode.Nullable) Field.build("foo", input, false) must beEqualTo(expected) } @@ -180,9 +180,9 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" |} """.stripMargin) - val expected = Field("foo",Type.Record(List( - Field("union",Type.String,Mode.Nullable) - )),Mode.Nullable) + val expected = Field("foo", Type.Record(List( + Field("union", Type.String, Mode.Nullable) + )), Mode.Nullable) Field.build("foo", input, false) must beEqualTo(expected) } @@ -201,7 +201,7 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" | } """.stripMargin) - val expected = Field("arrayTest",Type.Record(List(Field("imp",Type.String,Mode.Repeated))),Mode.Required) + val expected = Field("arrayTest", Type.Record(List(Field("imp", Type.String, Mode.Repeated))), Mode.Required) Field.build("arrayTest", input, true) must beEqualTo(expected) } @@ -219,7 +219,7 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" | } """.stripMargin) - val expected = Field("arrayTest",Type.Record(List(Field("imp",Type.String,Mode.Repeated))),Mode.Required) + val expected = Field("arrayTest", Type.Record(List(Field("imp", Type.String, Mode.Repeated))), Mode.Required) Field.build("arrayTest", input, true) must beEqualTo(expected) } From 3e785d167384e3c06be23467eb31e97d5d2f48ad Mon Sep 17 00:00:00 2001 From: "pavel.voropaev" Date: Thu, 27 Apr 2023 16:34:27 +0100 Subject: [PATCH 3/4] review feedback --- .../iglu.schemaddl/bigquery/Field.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Field.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Field.scala index 45298213..c09a3cf3 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Field.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Field.scala @@ -42,21 +42,21 @@ object Field { case Some(types) if types.possiblyWithNull(CommonProperties.Type.Object) => val subfields = topSchema.properties.map(_.value).getOrElse(Map.empty) if (subfields.isEmpty) { - Suggestion.finalSuggestion(topSchema, required || types.nullable)(name) + Suggestion.finalSuggestion(topSchema, required )(name) } else { val requiredKeys = topSchema.required.toList.flatMap(_.value) val fields = subfields.map { case (key, schema) => build(key, schema, requiredKeys.contains(key)) } val subFields = fields.toList.sortBy(field => (Mode.sort(field.mode), field.name)) - Field(name, Type.Record(subFields), Mode.required(required || types.nullable)) + Field(name, Type.Record(subFields), Mode.required(required )) } case Some(types) if types.possiblyWithNull(CommonProperties.Type.Array) => topSchema.items match { case Some(ArrayProperty.Items.ListItems(schema)) => build(name, schema, false).copy(mode = Mode.Repeated) case _ => - Suggestion.finalSuggestion(topSchema, required || types.nullable)(name) + Suggestion.finalSuggestion(topSchema, required )(name) } case _ => Suggestion.suggestions From eab84cfbd2394c2a851e86f8991358da8a36287d Mon Sep 17 00:00:00 2001 From: "pavel.voropaev" Date: Thu, 27 Apr 2023 22:30:44 +0100 Subject: [PATCH 4/4] review feedback 3 --- .../iglu.schemaddl/bigquery/Suggestion.scala | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala index fab73a58..2311b5d2 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala @@ -56,15 +56,7 @@ object Suggestion { Some(name => Field(name, numericWithMultiple(mult, schema.maximum, schema.minimum), Mode.required(required))) case None => None } - case Some(CommonProperties.Type.Union(types)) if withNull(types, CommonProperties.Type.Number) => - schema.multipleOf match { - case Some(NumberProperty.MultipleOf.IntegerMultipleOf(_)) => - Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.Nullable)) - case Some(mult: NumberProperty.MultipleOf.NumberMultipleOf) => - Some(name => Field(name, numericWithMultiple(mult, schema.maximum, schema.minimum), Mode.Nullable)) - case None => None - } - case Some(CommonProperties.Type.Union(types)) if onlyNumeric(types, true) => + case Some(CommonProperties.Type.Union(types)) if withNull(types, CommonProperties.Type.Number) | onlyNumeric(types, true) => schema.multipleOf match { case Some(NumberProperty.MultipleOf.IntegerMultipleOf(_)) => Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.Nullable))