Add SidecarTimeout and RemoteLoggingLevel to MeshConfig. (#186)

* Add SidecarTimeout to MeshConfig. * 支持SidecarTimeout参数设置sidecar的（connect/listen）各项超时参数 * Add RemoteLoggingLevel to MeshConfig. * Add RemoteLoggingLevel to MeshConfig. * Add RemoteLoggingLevel to MeshConfig. * 增加RemoteLoggingLevel配置项 * fix permissive e2e test. * fix permissive e2e test. * fix permissive e2e test. * fix permissive e2e test. * update version to 1.3.7 * 将tracing标记由x-b3改为x-b4 --------- Co-authored-by: wanpf <[email protected]>
flomesh-io · Sep 7, 2023 · 969c055 · 969c055
1 parent a920400
commit 969c055
Show file tree

Hide file tree

Showing 22 changed files with 195 additions and 40 deletions.
diff --git a/charts/osm/Chart.yaml b/charts/osm/Chart.yaml
@@ -14,11 +14,11 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 1.3.6
+version: 1.3.7
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: v1.3.6
+appVersion: v1.3.7
 
 # This specifies the minimum Kubernetes version OSM is compatible with.
 kubeVersion: ">= 1.19.0-0"

diff --git a/charts/osm/README.md b/charts/osm/README.md
@@ -1,6 +1,6 @@
 # Open Service Mesh Edge Helm Chart
 
-![Version: 1.3.6](https://img.shields.io/badge/Version-1.3.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.3.6](https://img.shields.io/badge/AppVersion-v1.3.6-informational?style=flat-square)
+![Version: 1.3.7](https://img.shields.io/badge/Version-1.3.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.3.7](https://img.shields.io/badge/AppVersion-v1.3.7-informational?style=flat-square)
 
 A Helm chart to install the [osm-edge](https://github.com/flomesh-io/osm-edge) control plane on Kubernetes.
 
@@ -150,7 +150,7 @@ The following table lists the configurable parameters of the osm chart and their
 | osm.image.name.osmSidecarInit | string | `"osm-edge-sidecar-init"` | Sidecar init container's image name |
 | osm.image.pullPolicy | string | `"IfNotPresent"` | Container image pull policy for control plane containers |
 | osm.image.registry | string | `"flomesh"` | Container image registry for control plane images |
-| osm.image.tag | string | `"1.3.6"` | Container image tag for control plane images |
+| osm.image.tag | string | `"1.3.7"` | Container image tag for control plane images |
 | osm.imagePullSecrets | list | `[]` | `osm-controller` image pull secret |
 | osm.inboundPortExclusionList | list | `[]` | Specifies a global list of ports to exclude from inbound traffic interception by the sidecar proxy. If specified, must be a list of positive integers. |
 | osm.injector.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"kubernetes.io/os"` |  |
@@ -327,6 +327,7 @@ The following table lists the configurable parameters of the osm chart and their
 | osm.remoteLogging.authorization | string | `""` | The authorization for remote logging service |
 | osm.remoteLogging.enable | bool | `false` | Toggles Sidecar's remote logging functionality on/off for all sidecar proxies in the mesh |
 | osm.remoteLogging.endpoint | string | `""` | Remote logging's API path where the spans will be sent to |
+| osm.remoteLogging.level | int | `2` | Level of the remote logging service |
 | osm.remoteLogging.port | int | `30514` | Port of the remote logging service |
 | osm.remoteLogging.sampledFraction | string | `"1.0"` | Sampled Fraction |
 | osm.repoServer | object | `{"codebase":"","image":"flomesh/pipy-repo:0.90.2-14","ipaddr":"127.0.0.1","standalone":false}` | Pipy RepoServer |
@@ -343,6 +344,7 @@ The following table lists the configurable parameters of the osm chart and their
 | osm.sidecarDrivers[1].sidecarWindowsImage | string | `"envoyproxy/envoy-windows:latest"` | Sidecar image for Windows workloads |
 | osm.sidecarImage | string | `""` | Sidecar image for Linux workloads |
 | osm.sidecarLogLevel | string | `"error"` | Log level for the proxy sidecar. Non developers should generally never set this value. In production environments the LogLevel should be set to `error` |
+| osm.sidecarTimeout | int | `60` | Sets connect/idle/read/write timeout |
 | osm.tracing.address | string | `""` | Address of the tracing collector service (must contain the namespace). When left empty, this is computed in helper template to "jaeger.<osm-namespace>.svc.cluster.local". Please override for BYO-tracing as documented in tracing.md |
 | osm.tracing.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"kubernetes.io/os"` |  |
 | osm.tracing.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator | string | `"In"` |  |

diff --git a/charts/osm/templates/preset-mesh-config.yaml b/charts/osm/templates/preset-mesh-config.yaml
@@ -14,6 +14,7 @@ data:
         "logLevel": {{.Values.osm.sidecarLogLevel | mustToJson}},
         "maxDataPlaneConnections": {{.Values.osm.maxDataPlaneConnections | mustToJson}},
         "configResyncInterval": {{.Values.osm.configResyncInterval | mustToJson}},
+        "sidecarTimeout": {{.Values.osm.sidecarTimeout | mustToJson}},
         "sidecarClass": {{.Values.osm.sidecarClass | mustToJson }},
         "sidecarImage": {{.Values.osm.sidecarImage | mustToJson }},
         "sidecarDrivers": {{.Values.osm.sidecarDrivers | mustToJson }},
@@ -45,6 +46,7 @@ data:
         "remoteLogging": {
           "enable": {{.Values.osm.remoteLogging.enable | mustToJson}}{{- if .Values.osm.remoteLogging.enable }},{{- end }}
           {{- if .Values.osm.remoteLogging.enable }}
+          "level": {{.Values.osm.remoteLogging.level | mustToJson}},
           "port": {{.Values.osm.remoteLogging.port | mustToJson}},
           "address": {{.Values.osm.remoteLogging.address | mustToJson}},
           "endpoint": {{.Values.osm.remoteLogging.endpoint | mustToJson}},

diff --git a/charts/osm/values.schema.json b/charts/osm/values.schema.json
@@ -990,6 +990,15 @@
                         "30s"
                     ]
                 },
+                "sidecarTimeout": {
+                    "$id": "#/properties/osm/properties/sidecarTimeout",
+                    "type": "integer",
+                    "title": "The sidecarTimeout schema",
+                    "description": "Sets connect/idle/read/write timeout",
+                    "examples": [
+                        "60"
+                    ]
+                },
                 "sidecarLogLevel": {
                     "$id": "#/properties/osm/properties/sidecarLogLevel",
                     "type": "string",
@@ -1177,6 +1186,7 @@
                     ],
                     "required": [
                         "enable",
+                        "level",
                         "address",
                         "port",
                         "endpoint"
@@ -1191,6 +1201,17 @@
                                 true
                             ]
                         },
+                        "level": {
+                            "$id": "#/properties/osm/properties/remoteLogging/properties/level",
+                            "type": "integer",
+                            "title": "The level schema for remote logging service",
+                            "description": "Level of the remote logging service",
+                            "minimum": 0,
+                            "maximum": 2,
+                            "examples": [
+                                2
+                            ]
+                        },
                         "address": {
                             "$id": "#/properties/osm/properties/remoteLogging/properties/address",
                             "type": "string",

diff --git a/charts/osm/values.yaml b/charts/osm/values.yaml
@@ -12,7 +12,7 @@ osm:
     # -- Container image pull policy for control plane containers
     pullPolicy: IfNotPresent
     # -- Container image tag for control plane images
-    tag: "1.3.6"
+    tag: "1.3.7"
     # -- Image name defaults
     name:
       # -- osm-controller's image name
@@ -449,6 +449,9 @@ osm:
   # -- Sets the resync interval for regular proxy broadcast updates, set to 0s to not enforce any resync
   configResyncInterval: "90s"
 
+  # -- Sets connect/idle/read/write timeout
+  sidecarTimeout: 60
+
   # -- Controller log verbosity
   controllerLogLevel: info
 
@@ -516,6 +519,8 @@ osm:
   remoteLogging:
     # -- Toggles Sidecar's remote logging functionality on/off for all sidecar proxies in the mesh
     enable: false
+    # -- Level of the remote logging service
+    level: 2
     # -- Address of the remote logging service (must contain the namespace). When left empty, this is computed in helper template to "remote-logging-service.<osm-namespace>.svc.cluster.local".
     address: ""
     # -- Port of the remote logging service

diff --git a/cmd/osm-bootstrap/crds/config_meshconfig.yaml b/cmd/osm-bootstrap/crds/config_meshconfig.yaml
@@ -138,6 +138,9 @@ spec:
                     configResyncInterval:
                       description: Resync interval for regular proxy broadcast updates
                       type: string
+                    sidecarTimeout:
+                      description: connect/idle/read/write timeout
+                      type: integer
                     tlsMinProtocolVersion:
                       description: The minimum TLS protocol version that the sidecar supports. Valid TLS protocol versions are TLS_AUTO, TLSv1_0, TLSv1_1, TLSv1_2 and TLSv1_3.
                       type: string
@@ -313,6 +316,11 @@ spec:
                         enable:
                           description: Enables remote logging for the mesh.
                           type: boolean
+                        level:
+                          description: Level on which remote logging is enabled.
+                          type: integer
+                          minimum: 0
+                          maximum: 2
                         port:
                           description: Port on which remote logging is enabled.
                           type: integer
@@ -565,6 +573,9 @@ spec:
                     configResyncInterval:
                       description: Resync interval for regular proxy broadcast updates
                       type: string
+                    sidecarTimeout:
+                      description: connect/idle/read/write timeout
+                      type: integer
                 repoServer:
                   description: Configuration for RepoServer
                   type: object
@@ -678,6 +689,11 @@ spec:
                         enable:
                           description: Enables remote logging for the mesh.
                           type: boolean
+                        level:
+                          description: Level on which remote logging is enabled.
+                          type: integer
+                          minimum: 0
+                          maximum: 2
                         port:
                           description: Port on which remote logging is enabled.
                           type: integer

diff --git a/pkg/apis/config/v1alpha1/mesh_config.go b/pkg/apis/config/v1alpha1/mesh_config.go
@@ -78,6 +78,9 @@ type SidecarSpec struct {
 	// ConfigResyncInterval defines the resync interval for regular proxy broadcast updates.
 	ConfigResyncInterval string `json:"configResyncInterval,omitempty"`
 
+	// SidecarTimeout defines the connect/idle/read/write timeout.
+	SidecarTimeout int `json:"sidecarTimeout,omitempty"`
+
 	// Resources defines the compute resources for the sidecar.
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 }
@@ -145,6 +148,9 @@ type RemoteLoggingSpec struct {
 	// Enable defines a boolean indicating if the sidecars are enabled for remote logging.
 	Enable bool `json:"enable"`
 
+	// Level defines the remote logging's level.
+	Level uint16 `json:"level,omitempty"`
+
 	// Port defines the remote loggings port.
 	Port uint16 `json:"port,omitempty"`
 

diff --git a/pkg/apis/config/v1alpha2/mesh_config.go b/pkg/apis/config/v1alpha2/mesh_config.go
@@ -100,6 +100,9 @@ type SidecarSpec struct {
 	// ConfigResyncInterval defines the resync interval for regular proxy broadcast updates.
 	ConfigResyncInterval string `json:"configResyncInterval,omitempty"`
 
+	// SidecarTimeout defines the connect/idle/read/write timeout.
+	SidecarTimeout int `json:"sidecarTimeout,omitempty"`
+
 	// Resources defines the compute resources for the sidecar.
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 
@@ -194,6 +197,9 @@ type RemoteLoggingSpec struct {
 	// Enable defines a boolean indicating if the sidecars are enabled for remote logging.
 	Enable bool `json:"enable"`
 
+	// Level defines the remote logging's level.
+	Level uint16 `json:"level,omitempty"`
+
 	// Port defines the remote logging's port.
 	Port int16 `json:"port,omitempty"`
 

diff --git a/pkg/configurator/methods.go b/pkg/configurator/methods.go
@@ -151,6 +151,11 @@ func (c *Client) IsRemoteLoggingEnabled() bool {
 	return c.getMeshConfig().Spec.Observability.RemoteLogging.Enable
 }
 
+// GetRemoteLoggingLevel returns the remote logging level
+func (c *Client) GetRemoteLoggingLevel() uint16 {
+	return c.getMeshConfig().Spec.Observability.RemoteLogging.Level
+}
+
 // GetRemoteLoggingHost is the host to which we send logging spans
 func (c *Client) GetRemoteLoggingHost() string {
 	remoteLoggingAddress := c.getMeshConfig().Spec.Observability.RemoteLogging.Address
@@ -203,6 +208,15 @@ func (c *Client) GetMaxDataPlaneConnections() int {
 	return c.getMeshConfig().Spec.Sidecar.MaxDataPlaneConnections
 }
 
+// GetSidecarTimeout returns connect/idle/read/write timeout
+func (c *Client) GetSidecarTimeout() int {
+	timeout := c.getMeshConfig().Spec.Sidecar.SidecarTimeout
+	if timeout <= 0 {
+		timeout = 60
+	}
+	return timeout
+}
+
 // GetSidecarLogLevel returns the sidecar log level
 func (c *Client) GetSidecarLogLevel() string {
 	logLevel := c.getMeshConfig().Spec.Sidecar.LogLevel

diff --git a/pkg/configurator/mock_client_generated.go b/pkg/configurator/mock_client_generated.go
diff --git a/pkg/configurator/types.go b/pkg/configurator/types.go
@@ -75,6 +75,9 @@ type Configurator interface {
 	// IsRemoteLoggingEnabled returns whether remote logging is enabled
 	IsRemoteLoggingEnabled() bool
 
+	// GetRemoteLoggingLevel returns the remote logging level
+	GetRemoteLoggingLevel() uint16
+
 	// GetRemoteLoggingHost is the host to which we send logging spans
 	GetRemoteLoggingHost() string
 
@@ -93,6 +96,9 @@ type Configurator interface {
 	// GetMaxDataPlaneConnections returns the max data plane connections allowed, 0 if disabled
 	GetMaxDataPlaneConnections() int
 
+	// GetSidecarTimeout returns connect/idle/read/write timeout
+	GetSidecarTimeout() int
+
 	// GetOSMLogLevel returns the configured OSM log level
 	GetOSMLogLevel() string
 

diff --git a/pkg/messaging/broker.go b/pkg/messaging/broker.go
@@ -342,7 +342,9 @@ func getProxyUpdateEvent(msg events.PubSubMessage) *proxyUpdateEvent {
 		if prevSpec.Traffic.EnableEgress != newSpec.Traffic.EnableEgress ||
 			prevSpec.Traffic.EnablePermissiveTrafficPolicyMode != newSpec.Traffic.EnablePermissiveTrafficPolicyMode ||
 			prevSpec.Observability.Tracing != newSpec.Observability.Tracing ||
+			prevSpec.Observability.RemoteLogging != newSpec.Observability.RemoteLogging ||
 			prevSpec.Sidecar.LogLevel != newSpec.Sidecar.LogLevel ||
+			prevSpec.Sidecar.SidecarTimeout != newSpec.Sidecar.SidecarTimeout ||
 			prevSpec.Traffic.InboundExternalAuthorization.Enable != newSpec.Traffic.InboundExternalAuthorization.Enable ||
 			// Only trigger an update on InboundExternalAuthorization field changes if the new spec has the 'Enable' flag set to true.
 			(newSpec.Traffic.InboundExternalAuthorization.Enable && (prevSpec.Traffic.InboundExternalAuthorization != newSpec.Traffic.InboundExternalAuthorization)) ||

diff --git a/pkg/sidecar/providers/pipy/driver/pipy_container.go b/pkg/sidecar/providers/pipy/driver/pipy_container.go
@@ -175,6 +175,10 @@ func getPipySidecarContainerSpec(injCtx *driver.InjectorContext, pod *corev1.Pod
 	}
 
 	if injCtx.Configurator.IsRemoteLoggingEnabled() {
+		sidecarContainer.Env = append(sidecarContainer.Env, corev1.EnvVar{
+			Name:  "REMOTE_LOGGING_LEVEL",
+			Value: fmt.Sprintf("%d", injCtx.Configurator.GetRemoteLoggingLevel()),
+		})
 		if len(injCtx.Configurator.GetRemoteLoggingHost()) > 0 && injCtx.Configurator.GetRemoteLoggingPort() > 0 {
 			sidecarContainer.Env = append(sidecarContainer.Env, corev1.EnvVar{
 				Name:  "REMOTE_LOGGING_ADDRESS",

diff --git a/pkg/sidecar/providers/pipy/repo/codebase/connect-tcp.js b/pkg/sidecar/providers/pipy/repo/codebase/connect-tcp.js
@@ -4,6 +4,16 @@
   {
     metricsCache,
   } = pipy.solve('metrics.js'),
+
+  connectOptions = (config?.Spec?.SidecarTimeout > 0) ? (
+    {
+      connectTimeout: config.Spec.SidecarTimeout,
+      readTimeout: config.Spec.SidecarTimeout,
+      writeTimeout: config.Spec.SidecarTimeout,
+      idleTimeout: config.Spec.SidecarTimeout,
+    }
+  ) : {},
+
 ) => (
 
 pipy({
@@ -44,10 +54,10 @@ pipy({
 )
 .branch(
   () => __target.startsWith('127.0.0.1:'), (
-    $=>$.connect(() => __target, { bind: '127.0.0.6' })
+    $=>$.connect(() => __target, { bind: '127.0.0.6', ...connectOptions})
   ),
   (
-    $=>$.connect(() => __target)
+    $=>$.connect(() => __target, connectOptions)
   )
 )
 .handleData(