Add Support for Relaxed Data Binding

ballerina/http_annotation.bal

@@ -27,6 +27,9 @@
 # + validation - Enables the inbound payload validation functionality which provided by the constraint package. Enabled by default
 # + serviceType - The service object type which defines the service contract. This is auto-generated at compile-time
 # + basePath - Base path to be used with the service implementation. This is only allowed on service contract types
+# + laxDataBinding - Enables or disables relaxed data binding on the service side. Disabled by default. 
+#                    When enabled, the JSON data will be projected to the Ballerina record type and during the projection,
+#                    nil values will be considered as optional fields and absent fields will be considered for nilable types
 public type HttpServiceConfig record {|
     string host = "b7a.default";
     CompressionConfig compression = {};
@@ -39,6 +42,7 @@ public type HttpServiceConfig record {|
     boolean validation = true;
     typedesc<ServiceContract> serviceType?;
     string basePath?;
+    boolean laxDataBinding = false;
 |};
 
 # Configurations for CORS support.

ballerina/http_client_endpoint.bal

@@ -28,13 +28,15 @@ import ballerina/time;
 #                HTTP service in resilient manner
 # + cookieStore - Stores the cookies of the client
 # + requireValidation - Enables the inbound payload validation functionalty which provided by the constraint package
+# + requireLaxDataBinding - Enables or disables relaxed data binding.
 public client isolated class Client {
     *ClientObject;
 
     private final string url;
     private CookieStore? cookieStore = ();
     final HttpClient httpClient;
     private final boolean requireValidation;
+    private final boolean requireLaxDataBinding;
 
     # Gets invoked to initialize the `client`. During initialization, the configurations provided through the `config`
     # record is used to determine which type of additional behaviours are added to the endpoint (e.g., caching,
@@ -53,6 +55,7 @@ public client isolated class Client {
         }
         self.httpClient = check initialize(url, config, self.cookieStore);
         self.requireValidation = config.validation;
+        self.requireLaxDataBinding = config.laxDataBinding;
         return;
     }
 
@@ -95,7 +98,7 @@ public client isolated class Client {
         if observabilityEnabled && response is Response {
             addObservabilityInformation(path, HTTP_POST, response.statusCode, self.url);
         }
-        return processResponse(response, targetType, self.requireValidation);
+        return processResponse(response, targetType, self.requireValidation, self.requireLaxDataBinding);
     }
 
     # The client resource function to send HTTP PUT requests to HTTP endpoints.
@@ -137,7 +140,7 @@ public client isolated class Client {
         if observabilityEnabled && response is Response {
             addObservabilityInformation(path, HTTP_PUT, response.statusCode, self.url);
         }
-        return processResponse(response, targetType, self.requireValidation);
+        return processResponse(response, targetType, self.requireValidation, self.requireLaxDataBinding);
     }
 
     # The client resource function to send HTTP PATCH requests to HTTP endpoints.
@@ -179,7 +182,7 @@ public client isolated class Client {
         if observabilityEnabled && response is Response {
             addObservabilityInformation(path, HTTP_PATCH, response.statusCode, self.url);
         }
-        return processResponse(response, targetType, self.requireValidation);
+        return processResponse(response, targetType, self.requireValidation, self.requireLaxDataBinding);
     }
 
     # The client resource function to send HTTP DELETE requests to HTTP endpoints.
@@ -221,7 +224,7 @@ public client isolated class Client {
         if observabilityEnabled && response is Response {
             addObservabilityInformation(path, HTTP_DELETE, response.statusCode, self.url);
         }
-        return processResponse(response, targetType, self.requireValidation);
+        return processResponse(response, targetType, self.requireValidation, self.requireLaxDataBinding);
     }
 
     # The client resource function to send HTTP HEAD requests to HTTP endpoints.
@@ -283,7 +286,7 @@ public client isolated class Client {
         if observabilityEnabled && response is Response {
             addObservabilityInformation(path, HTTP_GET, response.statusCode, self.url);
         }
-        return processResponse(response, targetType, self.requireValidation);
+        return processResponse(response, targetType, self.requireValidation, self.requireLaxDataBinding);
     }
 
     # The client resource function to send HTTP OPTIONS requests to HTTP endpoints.
@@ -319,7 +322,7 @@ public client isolated class Client {
         if observabilityEnabled && response is Response {
             addObservabilityInformation(path, HTTP_OPTIONS, response.statusCode, self.url);
         }
-        return processResponse(response, targetType, self.requireValidation);
+        return processResponse(response, targetType, self.requireValidation, self.requireLaxDataBinding);
     }
 
     # Invokes an HTTP call with the specified HTTP verb.
@@ -347,7 +350,7 @@ public client isolated class Client {
         if observabilityEnabled && response is Response {
             addObservabilityInformation(path, httpVerb, response.statusCode, self.url);
         }
-        return processResponse(response, targetType, self.requireValidation);
+        return processResponse(response, targetType, self.requireValidation, self.requireLaxDataBinding);
     }
 
     # The `Client.forward()` function can be used to invoke an HTTP call with inbound request's HTTP verb
@@ -368,7 +371,7 @@ public client isolated class Client {
         if observabilityEnabled && response is Response {
             addObservabilityInformation(path, request.method, response.statusCode, self.url);
         }
-        return processResponse(response, targetType, self.requireValidation);
+        return processResponse(response, targetType, self.requireValidation, self.requireLaxDataBinding);
     }
 
     # Submits an HTTP request to a service with the specified HTTP verb.
@@ -720,7 +723,7 @@ isolated function createStatusCodeResponseDataBindingError(DataBindingErrorType
     }
 }
 
-isolated function processResponse(Response|ClientError response, TargetType targetType, boolean requireValidation)
+isolated function processResponse(Response|ClientError response, TargetType targetType, boolean requireValidation, boolean requireLaxDataBinding)
         returns Response|stream<SseEvent, error?>|anydata|ClientError {
     if response is ClientError || hasHttpResponseType(targetType) {
         return response;
@@ -741,7 +744,7 @@ isolated function processResponse(Response|ClientError response, TargetType targ
         }
     }
     if targetType is typedesc<anydata> {
-        anydata payload = check performDataBinding(response, targetType);
+        anydata payload = check performDataBinding(response, targetType, requireLaxDataBinding);
         if requireValidation {
             return performDataValidation(payload, targetType);
         }
@@ -773,21 +776,21 @@ isolated function validateEventStreamContentType(Response response) returns Clie
     }
 }
 
-isolated function processResponseNew(Response|ClientError response, typedesc<StatusCodeResponse> targetType, boolean requireValidation)
-        returns StatusCodeResponse|ClientError {
+isolated function processResponseNew(Response|ClientError response, typedesc<StatusCodeResponse> targetType, boolean requireValidation, 
+    boolean requireLaxDataBinding) returns StatusCodeResponse|ClientError {
     if response is ClientError {
         return response;
     }
-    return externProcessResponseNew(response, targetType, requireValidation);
+    return externProcessResponseNew(response, targetType, requireValidation, requireLaxDataBinding);
 }
 
-isolated function externProcessResponse(Response response, TargetType targetType, boolean requireValidation)
+isolated function externProcessResponse(Response response, TargetType targetType, boolean requireValidation, boolean requireLaxDataBinding)
                                   returns Response|anydata|StatusCodeResponse|ClientError = @java:Method {
     'class: "io.ballerina.stdlib.http.api.nativeimpl.ExternResponseProcessor",
     name: "processResponse"
 } external;
 
-isolated function externProcessResponseNew(Response response, typedesc<StatusCodeResponse> targetType, boolean requireValidation)
+isolated function externProcessResponseNew(Response response, typedesc<StatusCodeResponse> targetType, boolean requireValidation, boolean requireLaxDataBinding)
                                   returns StatusCodeResponse|ClientError = @java:Method {
     'class: "io.ballerina.stdlib.http.api.nativeimpl.ExternResponseProcessor",
     name: "processResponse"

ballerina/http_client_payload_builder.bal

@@ -25,10 +25,10 @@ type stringType typedesc<string>;
 type byteArrType typedesc<byte[]>;
 type mapStringType typedesc<map<string>>;
 
-isolated function performDataBinding(Response response, TargetType targetType) returns anydata|ClientError {
+isolated function performDataBinding(Response response, TargetType targetType, boolean requireLaxDataBinding) returns anydata|ClientError {
     string contentType = response.getContentType().trim();
     if contentType == "" {
-        return getBuilderFromType(response, targetType);
+        return getBuilderFromType(response, targetType, requireLaxDataBinding);
     }
     if XML_PATTERN.isFullMatch(contentType) {
         return xmlPayloadBuilder(response, targetType);
@@ -39,13 +39,13 @@ isolated function performDataBinding(Response response, TargetType targetType) r
     } else if OCTET_STREAM_PATTERN.isFullMatch(contentType) {
         return blobPayloadBuilder(response, targetType);
     } else if JSON_PATTERN.isFullMatch(contentType) {
-        return jsonPayloadBuilder(response, targetType);
+        return jsonPayloadBuilder(response, targetType, requireLaxDataBinding);
     } else {
-        return getBuilderFromType(response, targetType);
+        return getBuilderFromType(response, targetType, requireLaxDataBinding);
     }
 }
 
-isolated function getBuilderFromType(Response response, TargetType targetType) returns anydata|ClientError {
+isolated function getBuilderFromType(Response response, TargetType targetType, boolean requireLaxDataBinding) returns anydata|ClientError {
     if targetType is typedesc<string> {
         return response.getTextPayload();
     } else if targetType is typedesc<string?> {
@@ -67,7 +67,7 @@ isolated function getBuilderFromType(Response response, TargetType targetType) r
     } else {
         // Due to the limitation of https://github.com/ballerina-platform/ballerina-spec/issues/1090
         // all the other types including union are considered as json subtypes.
-        return jsonPayloadBuilder(response, targetType);
+        return jsonPayloadBuilder(response, targetType, requireLaxDataBinding);
     }
 }
 
@@ -135,20 +135,20 @@ isolated function blobPayloadBuilder(Response response, TargetType targetType) r
     }
 }
 
-isolated function jsonPayloadBuilder(Response response, TargetType targetType) returns anydata|ClientError {
+isolated function jsonPayloadBuilder(Response response, TargetType targetType, boolean requireLaxDataBinding) returns anydata|ClientError {
     if targetType is typedesc<record {| anydata...; |}> {
-        return nonNilablejsonPayloadBuilder(response, targetType);
+        return nonNilablejsonPayloadBuilder(response, targetType, requireLaxDataBinding);
     } else if targetType is typedesc<record {| anydata...; |}?> {
-        return nilablejsonPayloadBuilder(response, targetType);
+        return nilablejsonPayloadBuilder(response, targetType, requireLaxDataBinding);
     } else if targetType is typedesc<record {| anydata...; |}[]> {
-        return nonNilablejsonPayloadBuilder(response, targetType);
+        return nonNilablejsonPayloadBuilder(response, targetType, requireLaxDataBinding);
     } else if targetType is typedesc<record {| anydata...; |}[]?> {
-        return nilablejsonPayloadBuilder(response, targetType);
+        return nilablejsonPayloadBuilder(response, targetType, requireLaxDataBinding);
     } else if targetType is typedesc<map<json>> {
         json payload = check response.getJsonPayload();
         return <map<json>> payload;
     } else if targetType is typedesc<anydata> {
-        return nilablejsonPayloadBuilder(response, targetType);
+        return nilablejsonPayloadBuilder(response, targetType, requireLaxDataBinding);
     } else {
         // Consume payload to avoid memory leaks
         byte[]|ClientError payload = response.getBinaryPayload();
@@ -159,18 +159,26 @@ isolated function jsonPayloadBuilder(Response response, TargetType targetType) r
     }
 }
 
-isolated function nonNilablejsonPayloadBuilder(Response response, typedesc<anydata> targetType)
+isolated function nonNilablejsonPayloadBuilder(Response response, typedesc<anydata> targetType, boolean requireLaxDataBinding)
         returns anydata|ClientError {
     json payload = check response.getJsonPayload();
-    var result = jsondata:parseAsType(payload, {enableConstraintValidation: false, allowDataProjection: false}, targetType);
+    jsondata:Options jsonParserOptions = {
+        enableConstraintValidation: false,
+        allowDataProjection: requireLaxDataBinding ? {nilAsOptionalField: true, absentAsNilableType: true} : false
+    };
+    var result = jsondata:parseAsType(payload, jsonParserOptions, targetType);
     return result is error ? createPayloadBindingError(result) : result;
 }
 
-isolated function nilablejsonPayloadBuilder(Response response, typedesc<anydata> targetType)
+isolated function nilablejsonPayloadBuilder(Response response, typedesc<anydata> targetType, boolean requireLaxDataBinding)
         returns anydata|ClientError {
     json|ClientError payload = response.getJsonPayload();
+    jsondata:Options jsonParserOptions = {
+        enableConstraintValidation: false,
+        allowDataProjection: requireLaxDataBinding ? {nilAsOptionalField: true, absentAsNilableType: true} : false
+    };
     if payload is json {
-        var result = jsondata:parseAsType(payload, {enableConstraintValidation: false, allowDataProjection: false}, targetType);
+        var result = jsondata:parseAsType(payload, jsonParserOptions, targetType);
         return result is error ? createPayloadBindingError(result) : result;
     } else {
         return payload is NoContentError ? () : payload;

ballerina/http_response.bal

@@ -586,10 +586,10 @@ public class Response {
     }
 
     isolated function buildStatusCodeResponse(typedesc<anydata>? payloadType, typedesc<StatusCodeResponse> statusCodeResType,
-        boolean requireValidation, Status status, map<string|int|boolean|string[]|int[]|boolean[]> headers, string? mediaType,
+        boolean requireValidation, boolean requireLaxDataBinding, Status status, map<string|int|boolean|string[]|int[]|boolean[]> headers, string? mediaType,
         boolean fromDefaultStatusCodeMapping) returns StatusCodeResponse|ClientError {
         if payloadType !is () {
-            anydata|ClientError payload = self.performDataBinding(payloadType, requireValidation);
+            anydata|ClientError payload = self.performDataBinding(payloadType, requireValidation, requireLaxDataBinding);
             if payload is ClientError {
                 return self.getStatusCodeResponseDataBindingError(payload.message(), fromDefaultStatusCodeMapping, PAYLOAD, payload);
             }
@@ -599,8 +599,8 @@ public class Response {
         }
     }
 
-    isolated function performDataBinding(typedesc<anydata> targetType, boolean requireValidation) returns anydata|ClientError {
-        anydata payload = check performDataBinding(self, targetType);
+    isolated function performDataBinding(typedesc<anydata> targetType, boolean requireValidation, boolean requireLaxDataBinding) returns anydata|ClientError {
+        anydata payload = check performDataBinding(self, targetType, requireLaxDataBinding);
         if requireValidation {
             return performDataValidation(payload, targetType);
         }