DALA-4584 Introduce Dataland Lego Bricks

.github/workflows/CI.yaml

@@ -210,7 +210,7 @@ jobs:
         run: |
           sudo apt update && sudo apt install libreoffice
           #TODO: look into a more elegant solution for the line below
-          ./gradlew :dataland-backend:test :dataland-backend-utils:test :dataland-internal-storage:test :dataland-api-key-manager:test :dataland-qa-service:test :dataland-document-manager:test :dataland-batch-manager:test :dataland-community-manager:test :dataland-email-service:test :dataland-framework-toolbox:test :dataland-external-storage:test :dataland-data-exporter:test --no-daemon --stacktrace
+          ./gradlew :dataland-backend:test :dataland-backend-utils:test :dataland-internal-storage:test :dataland-api-key-manager:test :dataland-qa-service:test :dataland-document-manager:test :dataland-batch-manager:test :dataland-community-manager:test :dataland-email-service:test :dataland-framework-toolbox:test :dataland-external-storage:test :dataland-data-exporter:test  :dataland-specification-service:test :dataland-specification-lib:test --no-daemon --stacktrace
         if: always()
         env:
           MAILJET_API_ID: ${{ secrets.MAILJET_API_ID }}

build-utils/base_rebuild_single_docker_image.sh

@@ -55,6 +55,7 @@ docker_build_args=(     --build-arg PROXY_ENVIRONMENT="${PROXY_ENVIRONMENT:-}" \
                         --build-arg DATALAND_INTERNAL_STORAGE_BASE_VERSION="${DATALAND_INTERNAL_STORAGE_BASE_VERSION:-}" \
                         --build-arg DATALAND_BATCH_MANAGER_BASE_VERSION="${DATALAND_BATCH_MANAGER_BASE_VERSION:-}" \
                         --build-arg DATALAND_COMMUNITY_MANAGER_BASE_VERSION="${DATALAND_COMMUNITY_MANAGER_BASE_VERSION:-}" \
+                        --build-arg DATALAND_SPECIFICATION_SERVICE_BASE_VERSION="${DATALAND_SPECIFICATION_SERVICE_BASE_VERSION:-}" \
                         --build-arg DATALAND_EMAIL_SERVICE_BASE_VERSION="${DATALAND_EMAIL_SERVICE_BASE_VERSION:-}" \
                         --build-arg DATALAND_EXTERNAL_STORAGE_BASE_VERSION="${DATALAND_EXTERNAL_STORAGE_BASE_VERSION:-}" \
                         --build-arg DATALAND_DUMMY_EURODAT_CLIENT_BASE_VERSION="${DATALAND_DUMMY_EURODAT_CLIENT_BASE_VERSION:-}" \

build-utils/rebuild_backend_images.sh

@@ -5,6 +5,8 @@ gradle_dependencies=$(grep gradle_dependencies ./build-utils/common.conf | cut -
 dependencies="./dataland-backend/ ./dataland-backend-utils/ ./dataland-keycloak-adapter/ ./dataland-message-queue-utils/ $gradle_dependencies"
 dependencies+=" ./dataland-api-key-manager/apiKeyManagerOpenApi.json ./dataland-community-manager/communityManagerOpenApi.json"
 dependencies+=" ./dataland-internal-storage/internalStorageOpenApi.json ./environments/.env.uncritical"
+dependencies+=" ./dataland-specification-service/specificationServiceOpenApi.json ./dataland-external-storage/externalStorageOpenApi.json"
+dependencies+=" ./dataland-document-manager/documentManagerOpenApi.json"
 
 ./build-utils/base_rebuild_single_docker_image.sh dataland_backend_base ./dataland-backend/DockerfileBase $dependencies
 

build-utils/rebuild_specification_service_images.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+gradle_dependencies=$(grep gradle_dependencies ./build-utils/common.conf | cut -d'=' -f2)
+dependencies="./dataland-specification-service/ ./dataland-backend-utils/ ./dataland-specification-lib/ $gradle_dependencies"
+
+./build-utils/base_rebuild_single_docker_image.sh dataland_specification_service_base ./dataland-specification-service/DockerfileBase $dependencies
+
+set -o allexport
+source ./*github_env.log
+set +o allexport
+
+./build-utils/base_rebuild_single_docker_image.sh dataland_specification_service_production ./dataland-specification-service/Dockerfile $dependencies
+
+./build-utils/base_rebuild_single_docker_image.sh dataland_specification_service_test ./dataland-specification-service/DockerfileTest $dependencies

dataland-automated-qa-service/src/main/infrastructure/resources/data_resource.py

@@ -60,9 +60,12 @@ def _load(self) -> None:
         token = get_access_token()
         backend_client = AuthenticatedClient(base_url=backend_api_url, token=token)
         logging.info(f"Retrieving meta information for dataset with ID {self.id}")
-        self.meta_info = get_data_meta_info(self.id, client=backend_client)
-        logging.info(f"Retrieving dataset with ID {self.id}")
-        self.data = _get_data(data_type=self.meta_info.data_type, data_id=self.id, client=backend_client).data
+        try:
+            self.meta_info = get_data_meta_info(self.id, client=backend_client)
+            logging.info(f"Retrieving dataset with ID {self.id}")
+            self.data = _get_data(data_type=self.meta_info.data_type, data_id=self.id, client=backend_client).data
+        except ValueError as e:
+            logging.info(f"Exception caught while retrieving metadata': {e}.")
 
 
 def _get_data(data_type: DataTypeEnum, data_id: str, client: AuthenticatedClient) -> any:
@@ -78,11 +81,13 @@ def _get_data(data_type: DataTypeEnum, data_id: str, client: AuthenticatedClient
         DataTypeEnum.HEIMATHAFEN: CompanyAssociatedDataHeimathafenData,
         DataTypeEnum.ADDITIONAL_COMPANY_INFORMATION: CompanyAssociatedDataAdditionalCompanyInformationData
     }
-    response = client.get_httpx_client().request(method="get", url=f"/data/{data_type}/{data_id}")
-    if response.status_code == HTTPStatus.OK:
-        return type_to_company_associated_data.get(data_type).from_dict(response.json())
-    if response.status_code == HTTPStatus.UNAUTHORIZED:
-        return cast(Any, None)
-    if client.raise_on_unexpected_status:
-        raise errors.UnexpectedStatus(response.status_code, response.content)
+
+    if type_to_company_associated_data.get(data_type):
+        response = client.get_httpx_client().request(method="get", url=f"/data/{data_type}/{data_id}")
+        if response.status_code == HTTPStatus.OK:
+            return type_to_company_associated_data.get(data_type).from_dict(response.json())
+        if response.status_code == HTTPStatus.UNAUTHORIZED:
+            return cast(Any, None)
+        if client.raise_on_unexpected_status:
+            raise errors.UnexpectedStatus(response.status_code, response.content)
     return None

dataland-backend-utils/src/main/kotlin/org/dataland/datalandbackendutils/exceptions/Constants.kt

@@ -2,3 +2,4 @@ package org.dataland.datalandbackendutils.exceptions
 
 const val SEARCHSTRING_TOO_SHORT_THRESHOLD = 3
 const val SEARCHSTRING_TOO_SHORT_VALIDATION_MESSAGE = "Search string is shorter than minimum number of characters"
+const val DATAPOINT_VALIDATION_CLASS_NOT_FOUND_MESSAGE = "Couldn't retrieve the validation class for the data point."

dataland-backend-utils/src/main/kotlin/org/dataland/datalandbackendutils/exceptions/ExceptionForwarder.kt

@@ -32,4 +32,25 @@ class ExceptionForwarder {
             )
         }
     }
+
+    /**
+     * Catches a bad-request-client-exception due to a validation error of a data point and throws a custom exception
+     * which is known to the ExceptionHandler.
+     * @param response the response in the inter-microservice-request that has a client error
+     * @param statusCode the status code of the client error
+     * @param throwable the client error itself
+     */
+    fun catchDataPointValidationClassNotFoundClientException(
+        response: String,
+        statusCode: Int,
+        throwable: Throwable,
+    ) {
+        if (statusCode == HttpStatus.NOT_FOUND.value() && response.contains(DATAPOINT_VALIDATION_CLASS_NOT_FOUND_MESSAGE)) {
+            throw InvalidInputApiException(
+                summary = "Failed to retrieve validation class for data point.",
+                message = DATAPOINT_VALIDATION_CLASS_NOT_FOUND_MESSAGE,
+                cause = throwable,
+            )
+        }
+    }
 }

dataland-backend-utils/src/main/kotlin/org/dataland/datalandbackendutils/exceptions/InternalServerErrorApiException.kt

@@ -4,13 +4,13 @@ import org.dataland.datalandbackendutils.model.ErrorDetails
 import org.springframework.http.HttpStatus
 
 /**
- * An InternalServerErrorApiException should be thrown if something went terribly wrong on Datalands side.
+ * An InternalServerErrorApiException should be thrown if something went terribly wrong on Dataland side.
  * It will be returned as a 500 error. publicSummary and publicMessage will be displayed to the user,
  * internalMessage and internalCause will only be displayed in development mode
  */
 class InternalServerErrorApiException(
-    val publicSummary: String,
-    val publicMessage: String,
+    private val publicSummary: String,
+    private val publicMessage: String,
     internalMessage: String,
     internalCause: Throwable? = null,
 ) : SingleApiException(internalMessage, internalCause) {

dataland-backend/DockerfileBase

@@ -10,6 +10,7 @@ COPY ./dataland-internal-storage/internalStorageOpenApi.json ./dataland-internal
 COPY ./dataland-external-storage/externalStorageOpenApi.json ./dataland-external-storage/externalStorageOpenApi.json
 COPY ./dataland-community-manager/communityManagerOpenApi.json ./dataland-community-manager/communityManagerOpenApi.json
 COPY ./dataland-document-manager/documentManagerOpenApi.json ./dataland-document-manager/documentManagerOpenApi.json
+COPY ./dataland-specification-service/specificationServiceOpenApi.json ./dataland-specification-service/specificationServiceOpenApi.json
 
 COPY ./dataland-keycloak-adapter/ ./dataland-keycloak-adapter/
 COPY ./.git/ ./.git

dataland-backend/backendOpenApi.json

@@ -964,6 +964,77 @@
         ]
       }
     },
+    "/data-points": {
+      "post": {
+        "tags": [
+          "data-point-controller"
+        ],
+        "summary": "Upload new data point.",
+        "description": "The uploaded data point is added to the data store, the generated data id is returned.",
+        "operationId": "postDataPoint",
+        "parameters": [
+          {
+            "name": "bypassQa",
+            "in": "query",
+            "required": false,
+            "schema": {
+              "type": "boolean",
+              "default": false
+            }
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/StorableDataPoint"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "200": {
+            "description": "Successfully added data to the data store.",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/DataPointMetaInformation"
+                }
+              }
+            }
+          },
+          "default": {
+            "description": "An error occurred",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ErrorResponse"
+                }
+              }
+            }
+          },
+          "401": {
+            "description": "Unauthorized",
+            "headers": {
+              "WWW-Authenticate": {
+                "schema": {
+                  "type": "string"
+                }
+              }
+            }
+          }
+        },
+        "security": [
+          {
+            "default-oauth": []
+          },
+          {
+            "default-bearer-auth": []
+          }
+        ]
+      }
+    },
     "/companies": {
       "get": {
         "tags": [
@@ -3154,6 +3225,66 @@
         ]
       }
     },
+    "/data-points/{dataId}": {
+      "get": {
+        "tags": [
+          "data-point-controller"
+        ],
+        "summary": "Retrieve data points by ID.",
+        "description": "A data point identified by its ID is retrieved.",
+        "operationId": "getDataPoint",
+        "parameters": [
+          {
+            "name": "dataId",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successfully retrieved data point.",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/StorableDataPoint"
+                }
+              }
+            }
+          },
+          "default": {
+            "description": "An error occurred",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ErrorResponse"
+                }
+              }
+            }
+          },
+          "401": {
+            "description": "Unauthorized",
+            "headers": {
+              "WWW-Authenticate": {
+                "schema": {
+                  "type": "string"
+                }
+              }
+            }
+          }
+        },
+        "security": [
+          {
+            "default-oauth": []
+          },
+          {
+            "default-bearer-auth": []
+          }
+        ]
+      }
+    },
     "/companies/{companyId}/info": {
       "get": {
         "tags": [
@@ -4072,6 +4203,11 @@
           },
           "fileReference": {
             "type": "string"
+          },
+          "publicationDate": {
+            "type": "string",
+            "format": "date",
+            "nullable": true
           }
         }
       },
@@ -4164,6 +4300,11 @@
           },
           "fileReference": {
             "type": "string"
+          },
+          "publicationDate": {
+            "type": "string",
+            "format": "date",
+            "nullable": true
           }
         }
       },
@@ -17630,6 +17771,63 @@
           }
         }
       },
+      "StorableDataPoint": {
+        "required": [
+          "companyId",
+          "dataPointContent",
+          "dataPointIdentifier",
+          "reportingPeriod"
+        ],
+        "type": "object",
+        "properties": {
+          "dataPointContent": {
+            "type": "string"
+          },
+          "dataPointIdentifier": {
+            "type": "string"
+          },
+          "companyId": {
+            "type": "string",
+            "format": "uuid"
+          },
+          "reportingPeriod": {
+            "type": "string"
+          }
+        }
+      },
+      "DataPointMetaInformation": {
+        "required": [
+          "companyId",
+          "dataId",
+          "dataPointIdentifier",
+          "reportingPeriod",
+          "uploadTime"
+        ],
+        "type": "object",
+        "properties": {
+          "dataId": {
+            "type": "string"
+          },
+          "dataPointIdentifier": {
+            "type": "string"
+          },
+          "companyId": {
+            "type": "string",
+            "format": "uuid"
+          },
+          "reportingPeriod": {
+            "type": "string"
+          },
+          "uploaderUserId": {
+            "type": "string",
+            "nullable": true
+          },
+          "uploadTime": {
+            "type": "integer",
+            "format": "int64"
+          }
+        }
+      },
       "CompanyInformationPatch": {
         "type": "object",
         "properties": {
@@ -17921,22 +18119,22 @@
         ],
         "type": "object",
         "properties": {
-          "sector": {
-            "type": "string",
-            "nullable": true
-          },
           "companyName": {
             "type": "string"
           },
           "companyId": {
             "type": "string"
           },
-          "countryCode": {
+          "headquarters": {
             "type": "string"
           },
-          "headquarters": {
+          "countryCode": {
             "type": "string"
           },
+          "sector": {
+            "type": "string",
+            "nullable": true
+          },
           "lei": {
             "type": "string",
             "nullable": true