Merge pull request #128 from open-runtimes/feat-tretry-0.6.x

cherry-pick: feat-retry-and-uniqid
open-runtimes · Oct 31, 2024 · 584934e · 584934e
2 parents 6a6140c + 2d48982
commit 584934e
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 9 deletions.
diff --git a/.env b/.env
@@ -11,3 +11,5 @@ OPR_EXECUTOR_LOGGING_CONFIG=
 OPR_EXECUTOR_DOCKER_HUB_USERNAME=
 OPR_EXECUTOR_DOCKER_HUB_PASSWORD=
 OPR_EXECUTOR_RUNTIME_VERSIONS=v4
+OPR_EXECUTOR_RETRY_ATTEMPTS=5
+OPR_EXECUTOR_RETRY_DELAY_MS=500
diff --git a/README.md b/README.md
@@ -60,6 +60,8 @@ services:
       - OPR_EXECUTOR_DOCKER_HUB_USERNAME
       - OPR_EXECUTOR_DOCKER_HUB_PASSWORD
       - OPR_EXECUTOR_RUNTIME_VERSIONS
+      - OPR_EXECUTOR_RETRY_ATTEMPTS
+      - OPR_EXECUTOR_RETRY_DELAY_MS
 
 networks:
   openruntimes-runtimes:
@@ -87,6 +89,8 @@ OPR_EXECUTOR_LOGGING_CONFIG=
 OPR_EXECUTOR_DOCKER_HUB_USERNAME=
 OPR_EXECUTOR_DOCKER_HUB_PASSWORD=
 OPR_EXECUTOR_RUNTIME_VERSIONS=v4
+OPR_EXECUTOR_RETRY_ATTEMPTS=5
+OPR_EXECUTOR_RETRY_DELAY_MS=500
 ```
 
 > `OPR_EXECUTOR_CONNECTION_STORAGE` takes a DSN string that represents a connection to your storage device. If you would like to use your local filesystem, you can use `file://localhost`. If using S3 or any other provider for storage, use a DSN of the following format `s3://access_key:access_secret@host:port/bucket_name?region=us-east-1`
@@ -196,6 +200,8 @@ docker compose down
 | OPR_EXECUTOR_DOCKER_HUB_USERNAME | Username for Docker Hub authentication (if applicable)                                                                                        |
 | OPR_EXECUTOR_DOCKER_HUB_PASSWORD | Password for Docker Hub authentication (if applicable)                                                                                        |
 | OPR_EXECUTOR_RUNTIME_VERSIONS    | Version tag for runtime environments, ex: `v4`                                                                                                |
+| OPR_EXECUTOR_RETRY_ATTEMPTS      | Number of retry attempts for failed executions, ex: `5`                                                                                       |
+| OPR_EXECUTOR_RETRY_DELAY_MS      | Delay (in milliseconds) between retry attempts, ex: `500`                                                                                    |
 
 ## Contributing
 

diff --git a/app/http.php b/app/http.php
@@ -490,7 +490,7 @@ function cleanUp(Orchestration $orchestration, Table $activeRuntimes, array $net
     ->action(function (string $runtimeId, string $image, string $entrypoint, string $source, string $destination, string $outputDirectory, array $variables, string $runtimeEntrypoint, string $command, int $timeout, bool $remove, float $cpus, int $memory, string $version, string $restartPolicy, array $networks, Orchestration $orchestration, Table $activeRuntimes, Response $response, Log $log) {
         $runtimeName = System::getHostname() . '-' . $runtimeId;
 
-        $runtimeHostname = \uniqid();
+        $runtimeHostname = \bin2hex(\random_bytes(16));
 
         $log->addTag('image', $image);
         $log->addTag('version', $version);
@@ -1272,23 +1272,43 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix
 
             // Execute function
             $executionRequest = $version === 'v4' ? $executeV4 : $executeV2;
-            $executionResponse = \call_user_func($executionRequest);
 
-            // Error occured
-            if ($executionResponse['errNo'] !== 0) {
-                // Intended timeout error for v2 functions
-                if ($executionResponse['errNo'] === 110 && $version === 'v2') {
-                    throw new Exception($executionResponse['error'], 400);
+            $retryDelayMs = \intval(Http::getEnv('OPR_EXECUTOR_RETRY_DELAY_MS', '500'));
+            $retryAttempts = \intval(Http::getEnv('OPR_EXECUTOR_RETRY_ATTEMPTS', '5'));
+
+            $attempts = 0;
+            do {
+                $executionResponse = \call_user_func($executionRequest);
+                if ($executionResponse['errNo'] === CURLE_OK) {
+                    break;
+                }
+
+                // Not retryable, return error immediately
+                if (!in_array($executionResponse['errNo'], [
+                    CURLE_COULDNT_RESOLVE_HOST, // 6
+                    CURLE_COULDNT_CONNECT, // 7
+                ])) {
+                    break;
                 }
 
+                usleep($retryDelayMs * 1000);
+            } while ((++$attempts < $retryAttempts) || (\microtime(true) - $startTime < $timeout));
+
+            // Error occurred
+            if ($executionResponse['errNo'] !== CURLE_OK) {
+                $log->addExtra('activeRuntime', $activeRuntimes->get($runtimeName));
                 $log->addExtra('error', $executionResponse['error']);
                 $log->addTag('hostname', $hostname);
 
-                throw new Exception('Internal curl errors has occurred within the executor! Error Number: ' . $executionResponse['errNo'], 500);
+                // Intended timeout error for v2 functions
+                if ($version === 'v2' && $executionResponse['errNo'] === SOCKET_ETIMEDOUT) {
+                    throw new Exception($executionResponse['error'], 400);
+                }
+
+                throw new Exception('Internal curl error has occurred within the executor! Error Number: ' . $executionResponse['errNo'], 500);
             }
 
             // Successful execution
-
             ['statusCode' => $statusCode, 'body' => $body, 'logs' => $logs, 'errors' => $errors, 'headers' => $headers] = $executionResponse;
 
             $endTime = \microtime(true);

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -44,6 +44,8 @@ services:
       - OPR_EXECUTOR_DOCKER_HUB_USERNAME
       - OPR_EXECUTOR_DOCKER_HUB_PASSWORD
       - OPR_EXECUTOR_RUNTIME_VERSIONS
+      - OPR_EXECUTOR_RETRY_ATTEMPTS
+      - OPR_EXECUTOR_RETRY_DELAY_MS
 
 volumes:
   openruntimes-builds: