unicode-org · echeran · Nov 20, 2024
diff --git a/common/testData/messageFormat/README.md b/common/testData/messageFormat/README.md
@@ -1,7 +1,3 @@
-# Test Data for CLDR MessageFormat 2.0 Tech Preview
-
-For information about MessageFormat 2.0, see [Unicode Locale Data Markup Language (LDML): Part 9: Message Format](../../../docs/ldml/tr35-messageFormat.md)
-
 The tests in the `./tests/` directory were originally copied from the [messageformat project](https://github.com/messageformat/messageformat/tree/11c95dab2b25db8454e49ff4daadb817e1d5b770/packages/mf2-messageformat/src/__fixtures)
 and are here relicensed by their original author (Eemeli Aro) under the Unicode License.
 
@@ -14,6 +10,8 @@ These test files are intended to be useful for testing multiple different messag
 - `data-model-errors.json` - Strings that should produce a Data Model Error when processed.
   Error names are defined in ["MessageFormat 2.0 Errors"](../spec/errors.md) in the spec.
 
+- `u-options.json` — Test cases for the `u:` options, using built-in functions.
+
 - `functions/` — Test cases that correspond to built-in functions.
   The behaviour of the built-in formatters is implementation-specific so the `exp` field is often
   omitted and assertions are made on error cases.
@@ -25,6 +23,7 @@ Some examples of test harnesses using these tests, from the source repository:
 - [Formatting tests](https://github.com/messageformat/messageformat/blob/11c95dab2b25db8454e49ff4daadb817e1d5b770/packages/mf2-messageformat/src/messageformat.test.ts)
 
 A [JSON schema](./schemas/) is included for the test files in this repository.
+
 ## Error Codes
 
 The following table relates the error names used in the [JSON schema](./schemas/)
@@ -88,8 +87,8 @@ its `Input`, `DecimalPlaces`, `FailsFormat`, and `FailsSelect` values are determ
 1. Let `DecimalPlaces` be 0.
 1. Let `FailsFormat` be `false`.
 1. Let `FailsSelect` be `false`.
-1. Let `arg` be the resolved value of the _expression_ _operand_.
-1. If `arg` is the resolved value of an _expression_
+1. Let `arg` be the _resolved value_ of the _expression_ _operand_.
+1. If `arg` is the _resolved value_ of an _expression_
    with a `:test:function`, `:test:select`, or `:test:format` _annotation_
    for which resolution has succeeded, then
    1. Let `Input` be the `Input` value of `arg`.
@@ -101,15 +100,15 @@ its `Input`, `DecimalPlaces`, `FailsFormat`, and `FailsSelect` values are determ
    1. Let `Input` be the numerical value of `arg`.
 1. Else,
    1. Emit "bad-input" _Resolution Error_.
-   1. Use a _fallback value_ as the resolved value of the _expression_.
+   1. Use a _fallback value_ as the _resolved value_ of the _expression_.
       Further steps of this algorithm are not followed.
 1. If the `decimalPlaces` _option_ is set, then
    1. If its value resolves to a numerical integer value 0 or 1
       or their corresponding string representations `'0'` or `'1'`, then
       1. Set `DecimalPlaces` to be the numerical value of the _option_.
    1. Else if its value is not an unresolved value set by _option resolution_,
       1. Emit "bad-option" _Resolution Error_.
-      1. Use a _fallback value_ as the resolved value of the _expression_.
+      1. Use a _fallback value_ as the _resolved value_ of the _expression_.
 1. If the `fails` _option_ is set, then
    1. If its value resolves to the string `'always'`, then
       1. Set `FailsFormat` to be `true`.
@@ -137,7 +136,7 @@ depends on its `Input`, `DecimalPlaces` and `FailsSelect` values.
 
 When an _expression_ with a `:test:function` _annotation_ is assigned to a _variable_ by a _declaration_
 and that _variable_ is used as an _option_ value,
-its resolved value is the `Input` value.
+its _resolved value_ is the `Input` value.
 
 When `:test:function` is used as a _formatter_,
 a _placeholder_ resolving to a value with a `:test:function` _expression_

diff --git a/common/testData/messageFormat/schemas/v0/tests.schema.json b/common/testData/messageFormat/schemas/v0/tests.schema.json
@@ -118,6 +118,9 @@
         "src": {
           "$ref": "#/$defs/src"
         },
+        "bidiIsolation": {
+          "$ref": "#/$defs/bidiIsolation"
+        },
         "params": {
           "$ref": "#/$defs/params"
         },
@@ -146,6 +149,9 @@
         "src": {
           "$ref": "#/$defs/src"
         },
+        "bidiIsolation": {
+          "$ref": "#/$defs/bidiIsolation"
+        },
         "params": {
           "$ref": "#/$defs/params"
         },
@@ -172,6 +178,10 @@
       "description": "The MF2 syntax source.",
       "type": "string"
     },
+    "bidiIsolation": {
+      "description": "The bidi isolation strategy.",
+      "enum": ["default", "none"]
+    },
     "params": {
       "description": "Parameters to pass in to the formatter for resolving external variables.",
       "type": "array",
@@ -243,6 +253,23 @@
               }
             }
           },
+          {
+            "description": "Bidi isolation part.",
+            "type": "object",
+            "additionalProperties": false,
+            "required": [
+              "type",
+              "value"
+            ],
+            "properties": {
+              "type": {
+                "const": "bidiIsolation"
+              },
+              "value": {
+                "enum": ["\u2066", "\u2067", "\u2068", "\u2069"]
+              }
+            }
+          },
           {
             "description": "Message markup part.",
             "type": "object",
@@ -269,6 +296,9 @@
               "name": {
                 "type": "string"
               },
+              "id": {
+                "type": "string"
+              },
               "options": {
                 "type": "object"
               }

diff --git a/common/testData/messageFormat/tests/bidi.json b/common/testData/messageFormat/tests/bidi.json
@@ -0,0 +1,146 @@
+{
+  "scenario": "Bidi support",
+  "description": "Tests for correct parsing of messages with bidirectional marks and isolates",
+  "defaultTestProperties": {
+    "bidiIsolation": "default",
+    "locale": "en-US"
+  },
+  "tests": [
+    {
+        "description": "simple-message    = o [simple-start pattern]",
+        "src": "  \u061C Hello world!",
+        "exp": "  \u061C Hello world!"
+    },
+    {
+        "description": "complex-message   = o *(declaration o) complex-body o",
+        "src": "\u200E .local $x = {1} {{ {$x}}}",
+        "exp": " \u20681\u2069"
+    },
+    {
+        "description": "complex-message   = o *(declaration o) complex-body o",
+        "src": ".local $x = {1} \u200F {{ {$x}}}",
+        "exp": " \u20681\u2069"
+    },
+    {
+        "description": "complex-message   = o *(declaration o) complex-body o",
+        "src": ".local $x = {1} {{ {$x}}} \u2066",
+        "exp": " \u20681\u2069"
+    },
+    {
+        "description": "input-declaration = input o variable-expression",
+        "src": ".input \u2067 {$x :number} {{hello}}",
+        "params": [{"name": "x", "value": "1"}],
+        "exp": "hello"
+    },
+    {
+        "description": "local s variable o \"=\" o expression",
+        "src": ".local $x \u2068 = \u2069 {1} {{hello}}",
+        "exp": "hello"
+    },
+    {
+        "description": "local s variable o \"=\" o expression",
+        "src": ".local \u2067 $x = {1} {{hello}}",
+        "exp": "hello"
+    },
+    {
+        "description": "local s variable o \"=\" o expression",
+        "src": ".local\u2067 $x = {1} {{hello}}",
+        "exp": "hello"
+    },
+    {
+        "description": "o \"{{\" pattern \"}}\"",
+        "src": "\u2067 {{hello}}",
+        "exp": "hello"
+    },
+    {
+        "description": "match-statement s variant *(o variant)",
+        "src": ".local $x = {1 :number}\n.match $x\n1 {{one}}\n\u061C * {{other}}",
+        "exp": "one"
+    },
+    {
+        "description": "match-statement s variant *(o variant)",
+        "src": ".local $x = {1 :number}.match $x \u061c1 {{one}}* {{other}}",
+        "exp": "one"
+    },
+    {
+        "description": "match-statement s variant *(o variant)",
+        "src": ".local $x = {1 :number}.match $x\u061c1 {{one}}* {{other}}",
+        "expErrors": [{"type": "syntax-error"}]
+    },
+    {
+        "description": "variant = key *(s key) quoted-pattern",
+        "src": ".local $x = {1 :number} .local $y = {$x :number}.match $x $y\n1 \u200E 1 {{one}}* * {{other}}",
+        "exp": "one"
+    },
+    {
+        "description": "variant = key *(s key) quoted-pattern",
+        "src": ".local $x = {1 :number} .local $y = {$x :number}.match $x $y\n1\u200E 1 {{one}}* * {{other}}",
+        "exp": "one"
+    },
+    {
+        "description": "literal-expression  = \"{\" o literal [s function] *(s attribute) o \"}\"",
+        "src": "{\u200E hello \u200F}",
+        "exp": "\u2068hello\u2069"
+    },
+    {
+        "description": "variable-expression = \"{\" o variable [s function] *(s attribute) o \"}\"",
+        "src": ".local $x = {1} {{ {\u200E $x \u200F} }}",
+        "exp": " \u20681\u2069 "
+    },
+    {
+        "description": "function-expression = \"{\" o function *(s attribute) o \"}\"",
+        "src": "{1 \u200E :number \u200F}",
+        "exp": "1"
+    },
+    {
+        "description": "markup = \"{\" o \"#\" identifier *(s option) *(s attribute) o [\"/\"] \"}\"",
+        "src": "{\u200F #b \u200E }",
+        "exp": ""
+    },
+    {
+        "description": "markup = \"{\" o \"/\" identifier *(s option) *(s attribute) o \"}\"",
+        "src": "{\u200F /b \u200E }",
+        "exp": ""
+    },
+    {
+        "description": "option = identifier o \"=\" o (literal / variable)",
+        "src": "{1 :number minimumFractionDigits\u200F=\u200E1 }",
+        "exp": "1.0"
+    },
+    {
+        "description": "attribute      = \"@\" identifier [o \"=\" o (literal / variable)]",
+        "src": "{1 :number @locale\u200F=\u200Een }",
+        "exp": "1"
+    },
+    {
+        "description":  " name... excludes U+FFFD and U+061C -- this pases as name -> [bidi] name-start *name-char",
+        "src": ".local $\u061Cfoo = {1} {{ {$\u061Cfoo} }}",
+        "exp": " \u20681\u2069 "
+    },
+    {
+        "description":  " name matches https://www.w3.org/TR/REC-xml-names/#NT-NCName but excludes U+FFFD and U+061C",
+        "src": ".local $foo\u061Cbar = {2} {{ }}",
+        "expErrors": [{"type": "syntax-error"}]
+    },
+    {
+        "description":  "name       = [bidi] name-start *name-char [bidi]",
+        "src": ".local $\u200Efoo\u200F = {3} {{{$\u200Efoo\u200F}}}",
+        "exp": "\u20683\u2069"
+    },
+    {
+        "description":  "name       = [bidi] name-start *name-char [bidi]",
+        "src": ".local $foo = {4} {{{$\u200Efoo\u200F}}}",
+        "exp": "\u20684\u2069"
+    },
+    {
+        "description":  "name       = [bidi] name-start *name-char [bidi]",
+        "src": ".local $\u200Efoo\u200F = {5} {{{$foo}}}",
+        "exp": "\u20685\u2069"
+    },
+    {
+        "description":  "name       = [bidi] name-start *name-char [bidi]",
+        "src": ".local $foo\u200Ebar = {5} {{{$foo\u200Ebar}}}",
+        "expErrors": [{"type": "syntax-error"}]
+    }
+  ]
+}
diff --git a/common/testData/messageFormat/tests/fallback.json b/common/testData/messageFormat/tests/fallback.json
@@ -0,0 +1,52 @@
+{
+  "$schema": "https://raw.githubusercontent.com/unicode-org/message-format-wg/main/test/schemas/v0/tests.schema.json",
+  "scenario": "Fallback",
+  "description": "Test cases for fallback behaviour.",
+  "defaultTestProperties": {
+    "bidiIsolation": "none",
+    "locale": "en-US",
+    "expErrors": true
+  },
+  "tests": [
+    {
+      "description": "function with unquoted literal operand",
+      "src": "{42 :test:function fails=format}",
+      "exp": "{|42|}"
+    },
+    {
+      "description": "function with quoted literal operand",
+      "src": "{|C:\\\\| :test:function fails=format}",
+      "exp": "{|C:\\\\|}"
+    },
+    {
+      "description": "unannotated implicit input variable",
+      "src": "{$var}",
+      "exp": "{$var}"
+    },
+    {
+      "description": "annotated implicit input variable",
+      "src": "{$var :number}",
+      "exp": "{$var}"
+    },
+    {
+      "description": "local variable with unknown function in declaration",
+      "src": ".local $var = {|val| :test:undefined} {{{$var}}}",
+      "exp": "{$var}"
+    },
+    {
+      "description": "function with local variable operand with unknown function in declaration",
+      "src": ".local $var = {|val| :test:undefined} {{{$var :test:function}}}",
+      "exp": "{$var}"
+    },
+    {
+      "description": "local variable with unknown function in placeholder",
+      "src": ".local $var = {|val|} {{{$var :test:undefined}}}",
+      "exp": "{$var}"
+    },
+    {
+      "description": "function with no operand",
+      "src": "{:test:undefined}",
+      "exp": "{:test:undefined}"
+    }
+  ]
+}
diff --git a/common/testData/messageFormat/tests/functions/currency.json b/common/testData/messageFormat/tests/functions/currency.json
@@ -0,0 +1,61 @@
+{
+  "$schema": "https://raw.githubusercontent.com/unicode-org/message-format-wg/main/test/schemas/v0/tests.schema.json",
+  "scenario": "Currency function",
+  "description": "The built-in formatter and selector for currencies.",
+  "defaultTestProperties": {
+    "bidiIsolation": "none",
+    "locale": "en-US"
+  },
+  "tests": [
+    {
+      "src": "{:currency}",
+      "expErrors": [{ "type": "bad-operand" }]
+    },
+    {
+      "src": "{foo :currency}",
+      "expErrors": [{ "type": "bad-operand" }]
+    },
+    {
+      "src": "{42 :currency}",
+      "expErrors": [{ "type": "bad-operand" }]
+    },
+    {
+      "src": ".local $n = {42 :number} {{{$n :currency}}}",
+      "expErrors": [{ "type": "bad-operand" }]
+    },
+    {
+      "src": "{42 :currency currency=EUR}",
+      "expErrors": false
+    },
+    {
+      "src": ".local $n = {42 :number} {{{$n :currency currency=EUR}}}",
+      "expErrors": false
+    },
+    {
+      "src": ".local $n = {42 :integer} {{{$n :currency currency=EUR}}}",
+      "expErrors": false
+    },
+    {
+      "src": ".local $n = {42 :currency currency=EUR} {{{$n :currency}}}",
+      "expErrors": false
+    },
+    {
+      "src": "{42 :currency currency=EUR fractionDigits=auto}",
+      "expErrors": false
+    },
+    {
+      "src": "{42 :currency currency=EUR fractionDigits=2}",
+      "expErrors": false
+    },
+    {
+      "src": "{$x :currency currency=EUR}",
+      "params": [{ "name": "x", "value": 41 }],
+      "expErrors": false
+    },
+    {
+      "src": ".local $n = {42 :currency currency=EUR} .match $n * {{other}}",
+      "exp": "other",
+      "expErrors": false
+    }
+  ]
+}