Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Common: Upgrade yauaa to 7.x #678

Merged
merged 1 commit into from
Nov 8, 2022
Merged

Common: Upgrade yauaa to 7.x #678

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 36 additions & 34 deletions .../nl.basjes/yauaa_context/jsonschema/1-0-3 → .../nl.basjes/yauaa_context/jsonschema/1-0-4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,129 +5,131 @@
"vendor": "nl.basjes",
"name": "yauaa_context",
"format": "jsonschema",
"version": "1-0-3"
"version": "1-0-4"
},
"type": "object",
"properties": {
"deviceClass": {
"description": "See https://yauaa.basjes.nl/README-Output.html",
"enum": ["Desktop", "Anonymized", "Unknown", "UNKNOWN", "Mobile", "Tablet", "Phone", "Watch", "Virtual Reality", "eReader", "Set-top box", "TV", "Game Console", "Handheld Game Console", "Voice", "Robot", "Robot Mobile", "Spy", "Hacker", "Augmented Reality", "Robot Imitator"]
"enum": ["Desktop", "Anonymized", "Unknown", "UNKNOWN", "Mobile", "Tablet", "Phone", "Watch", "Virtual Reality", "eReader", "Set-top box", "TV", "Game Console", "Home Appliance", "Handheld Game Console", "Voice", "Car", "Robot", "Robot Mobile", "Spy", "Hacker", "Augmented Reality", "Robot Imitator"]
},
"deviceName": {
"description": "Example: Google Nexus 6",
"type": "string",
"maxLength": 100
"maxLength": 256
},
"deviceBrand": {
"description": "Example: Google",
"type": "string",
"maxLength": 50
"maxLength": 128
},
"deviceCpu": {
"type": "string",
"maxLength": 50
"maxLength": 128
},
"deviceCpuBits": {
"type": "string",
"maxLength": 20
"maxLength": 128
},
"deviceFirmwareVersion": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"deviceVersion": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"operatingSystemClass": {
"description": "See https://yauaa.basjes.nl/README-Output.html",
"type": "string",
"enum": ["Desktop", "Mobile", "Cloud", "Embedded", "Game Console", "Hacker", "Anonymized", "Unknown"]
},
"operatingSystemName": {
"description": "Examples: Linux, Android.",
"type": "string",
"maxLength": 100
"maxLength": 256
},
"operatingSystemVersion": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"operatingSystemNameVersion": {
"type": "string",
"maxLength": 150
"maxLength": 1000
},
"operatingSystemVersionBuild": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"layoutEngineClass": {
"description": "See https://yauaa.basjes.nl/README-Output.html",
"type": "string",
"enum": ["Browser", "Mobile App", "Hacker", "Robot", "Unknown", "Special", "Cloud", "eReader"]
},
"layoutEngineName": {
"type": "string",
"maxLength": 100
"maxLength": 256
},
"layoutEngineVersion": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"layoutEngineVersionMajor": {
"type": "string",
"maxLength": 20
"maxLength": 1000
},
"layoutEngineNameVersion": {
"type": "string",
"maxLength": 150
"maxLength": 1000
},
"layoutEngineNameVersionMajor": {
"type": "string",
"maxLength": 120
"maxLength": 1000
},
"layoutEngineBuild": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"agentClass": {
"description": "See https://yauaa.basjes.nl/README-Output.html",
"type": "string",
"enum": ["Browser", "Browser Webview", "Mobile App", "Robot", "Robot Mobile", "Cloud Application", "Email Client", "Voice", "Special", "Testclient", "Hacker", "Unknown", "Desktop App", "eReader"]
},
"agentName": {
"description": "Example: Chrome.",
"type": "string",
"maxLength": 100
"maxLength": 256
},
"agentVersion": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"agentVersionMajor": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"agentNameVersion": {
"type": "string",
"maxLength": 200
"maxLength": 1000
},
"agentNameVersionMajor": {
"type": "string",
"maxLength": 120
"maxLength": 1000
},
"agentBuild": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"agentLanguage": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"agentLanguageCode": {
"type": "string",
"maxLength": 20
"maxLength": 1000
},
"agentInformationEmail": {
"type": "string",
"format": "email"
"type": "string"
},
"agentInformationUrl": {
"type": "string"
Expand All @@ -147,11 +149,11 @@
},
"webviewAppVersionMajor": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"webviewAppNameVersionMajor": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"facebookCarrier": {
"type": "string"
Expand Down Expand Up @@ -196,19 +198,19 @@
},
"iECompatibilityVersion": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"iECompatibilityVersionMajor": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"iECompatibilityNameVersion": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"iECompatibilityNameVersionMajor": {
"type": "string",
"maxLength": 70
"maxLength": 1000
},
"carrier": {
"type": "string"
Expand Down
54 changes: 52 additions & 2 deletions ...owplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/YauaaEnrichmentSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,65 @@ class YauaaEnrichmentSpec extends Specification with CatsIO {
val input = BlackBoxTesting.buildCollectorPayload(
path = "/i",
querystring = "e=pp".some,
userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0".some
userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36".some,
headers = List(
"""Sec-CH-UA: "Chromium";v="106", "Google Chrome";v="106", "Not;A=Brand";v="99"""",
"""Sec-CH-UA-Arch: "x86"""",
"""Sec-CH-UA-Bitness: "64"""",
"""Sec-CH-UA-Full-Version: "106.0.5249.119"""",
"""Sec-CH-UA-Full-Version-List: "Chromium";v="106.0.5249.119", "Google Chrome";v="106.0.5249.119", "Not;A=Brand";v="99.0.0.0"""",
"""Sec-CH-UA-Mobile: ?0""",
"""Sec-CH-UA-Model: """"",
"""Sec-CH-UA-Platform: "Linux"""",
"""Sec-CH-ua-Platform-version: "6.0.1"""",
"""Sec-CH-UA-WoW64: ?0"""
)
)
val expected = Map(
"event_vendor" -> "com.snowplowanalytics.snowplow",
"event_name" -> "page_ping",
"event_format" -> "jsonschema",
"event_version" -> "1-0-0",
"event" -> "page_ping",
"derived_contexts" -> json"""{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-3","data":{"deviceBrand":"Unknown","deviceName":"Desktop","operatingSystemVersionMajor":"7","layoutEngineNameVersion":"Gecko 12.0","operatingSystemNameVersion":"Windows 7","layoutEngineBuild":"20100101","layoutEngineNameVersionMajor":"Gecko 12","operatingSystemName":"Windows NT","agentVersionMajor":"12","layoutEngineVersionMajor":"12","deviceClass":"Desktop","agentNameVersionMajor":"Firefox 12","operatingSystemNameVersionMajor":"Windows 7","deviceCpuBits":"64","operatingSystemClass":"Desktop","layoutEngineName":"Gecko","agentName":"Firefox","agentVersion":"12.0","layoutEngineClass":"Browser","agentNameVersion":"Firefox 12.0","operatingSystemVersion":"7","deviceCpu":"Intel x86_64","agentClass":"Browser","layoutEngineVersion":"12.0"}}]}""".noSpaces
"derived_contexts" -> json"""{
"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1",
"data":[{
"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-4",
"data": {
"deviceBrand":"Unknown",
"deviceName":"Linux Desktop",
"operatingSystemVersionMajor":"6",
"layoutEngineNameVersion":"Blink 106.0",
"operatingSystemNameVersion":"Linux 6.0.1",
"agentInformationEmail": "Unknown",
"networkType": "Unknown",
"operatingSystemVersionBuild":"??",
"webviewAppNameVersionMajor": "Unknown ??",
"layoutEngineNameVersionMajor":"Blink 106",
"operatingSystemName":"Linux",
"agentVersionMajor":"106",
"layoutEngineVersionMajor":"106",
"webviewAppName": "Unknown",
"deviceClass":"Desktop",
"agentNameVersionMajor":"Chrome 106",
"operatingSystemNameVersionMajor":"Linux 6",
"deviceCpuBits":"64",
"webviewAppVersionMajor": "??",
"operatingSystemClass":"Desktop",
"webviewAppVersion": "??",
"layoutEngineName":"Blink",
"agentName":"Chrome",
"agentVersion":"106.0.5249.119",
"layoutEngineClass":"Browser",
"agentNameVersion":"Chrome 106.0.5249.119",
"operatingSystemVersion":"6.0.1",
"deviceCpu":"Intel x86_64",
"agentClass":"Browser",
"layoutEngineVersion":"106.0",
"agentInformationUrl": "Unknown"
}
}]
}""".noSpaces
)
BlackBoxTesting.runTest(input, expected, Some(YauaaEnrichmentSpec.conf))
}
Expand Down
18 changes: 12 additions & 6 deletions ...la/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/YauaaEnrichmentSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,23 +53,29 @@ class YauaaEnrichmentSpec extends Specification with CatsIO {
val expected = Contexts(
List(
SelfDescribingData(
SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(1, 0, 3)),
SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(1, 0, 4)),
json"""{
"deviceBrand" : "Apple",
"deviceName" : "Apple Macintosh",
"operatingSystemVersionMajor" : "10",
"operatingSystemVersionMajor" : "10.14",
"layoutEngineNameVersion" : "Gecko 81.0",
"operatingSystemNameVersion" : "Mac OS X 10.14",
"operatingSystemNameVersion" : "Mac OS 10.14",
"agentInformationEmail": "Unknown",
"networkType": "Unknown",
"layoutEngineBuild" : "20100101",
"webviewAppNameVersionMajor": "Unknown ??",
"layoutEngineNameVersionMajor" : "Gecko 81",
"operatingSystemName" : "Mac OS X",
"operatingSystemName" : "Mac OS",
"agentVersionMajor" : "81",
"layoutEngineVersionMajor" : "81",
"webviewAppName": "Unknown",
"deviceClass" : "Desktop",
"agentNameVersionMajor" : "Firefox 81",
"operatingSystemNameVersionMajor" : "Mac OS X 10",
"deviceCpuBits" : "32",
"operatingSystemNameVersionMajor" : "Mac OS 10.14",
"deviceCpuBits" : "64",
"webviewAppVersionMajor": "??",
"operatingSystemClass" : "Desktop",
"webviewAppVersion": "??",
"layoutEngineName" : "Gecko",
"agentName" : "Firefox",
"agentVersion" : "81.0",
Expand Down
8 changes: 4 additions & 4 deletions ...in/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ object EnrichmentManager {
_ <- setEventFingerprint[F](raw.parameters, registry.eventFingerprint) // This enrichment cannot fail
_ <- getCookieContexts // Execute cookie extractor enrichment
_ <- getHttpHeaderContexts // Execute header extractor enrichment
_ <- getYauaaContext[F](registry.yauaa) // Runs YAUAA enrichment (gets info thanks to user agent)
_ <- getYauaaContext[F](registry.yauaa, raw.context.headers) // Runs YAUAA enrichment (gets info thanks to user agent)
_ <- extractSchemaFields[F](unstructEvent) // Extract the event vendor/name/format/version
_ <- getJsScript[F](registry.javascriptScript) // Execute the JavaScript scripting enrichment
_ <- getCurrency[F](raw.context.timestamp, registry.currencyConversion) // Finalize the currency conversion
Expand Down Expand Up @@ -237,7 +237,7 @@ object EnrichmentManager {
_ <- getCookieContexts // Execute cookie extractor enrichment
_ <- getHttpHeaderContexts // Execute header extractor enrichment
_ <- getWeatherContext[F](registry.weather) // Fetch weather context
_ <- getYauaaContext[F](registry.yauaa) // Runs YAUAA enrichment (gets info thanks to user agent)
_ <- getYauaaContext[F](registry.yauaa, raw.context.headers) // Runs YAUAA enrichment (gets info thanks to user agent)
_ <- extractSchemaFields[F](unstructEvent) // Extract the event vendor/name/format/version
_ <- geoLocation[F](registry.ipLookups) // Execute IP lookup enrichment
_ <- getJsScript[F](registry.javascriptScript) // Execute the JavaScript scripting enrichment
Expand Down Expand Up @@ -693,10 +693,10 @@ object EnrichmentManager {
}
}

def getYauaaContext[F[_]: Applicative](yauaa: Option[YauaaEnrichment]): EStateT[F, Unit] =
def getYauaaContext[F[_]: Applicative](yauaa: Option[YauaaEnrichment], headers: List[String]): EStateT[F, Unit] =
EStateT.fromEither {
case (event, _) =>
yauaa.map(_.getYauaaContext(event.useragent)).toList.asRight
yauaa.map(_.getYauaaContext(event.useragent, headers)).toList.asRight
}

// Derive some contexts with custom SQL Query enrichment
Expand Down
Loading