Implemented possibility for configuring traffic splitting, and fallba…

…ck using aggregate cluster #292
allegro · Aug 24, 2023 · 2865fb2 · 2865fb2
1 parent cdf8d9c
commit 2865fb2
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 87 deletions.
diff --git a/.../allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt b/.../allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt
@@ -188,7 +188,7 @@ class EnvoyClustersFactory(
         return emptyList()
     }
 
-    // todo AD refactor
+    // todo refactor
     private fun getEdsClustersForGroup(group: Group, globalSnapshot: GlobalSnapshot): List<Cluster> {
         val clusters: Map<String, Cluster> = if (enableTlsForGroup(group)) {
             globalSnapshot.securedClusters
@@ -217,7 +217,7 @@ class EnvoyClustersFactory(
                         } else group.proxySettings.outgoing.defaultServiceSettings
                     createClusters(
                         group.serviceName,
-                        dependencies.keys,
+                        globalSnapshot.allServicesNames,
                         dependencySettings,
                         clusters[it],
                         globalSnapshot.endpoints[it]

diff --git a/...core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt b/...core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt
@@ -15,7 +15,6 @@ import io.envoyproxy.envoy.config.listener.v3.Listener
 import io.micrometer.core.instrument.MeterRegistry
 import io.micrometer.core.instrument.simple.SimpleMeterRegistry
 import org.assertj.core.api.Assertions.assertThat
-import org.junit.Ignore
 import org.junit.jupiter.api.Test
 import pl.allegro.tech.servicemesh.envoycontrol.groups.AccessLogFilterSettings
 import pl.allegro.tech.servicemesh.envoycontrol.groups.AllServicesGroup
@@ -59,7 +58,7 @@ class EnvoySnapshotFactoryTest {
         const val CLUSTER_NAME = "cluster-name"
         const val MAIN_CLUSTER_NAME = "service-name-2"
         const val SECONDARY_CLUSTER_NAME = "service-name-2-secondary"
-        const val AGGREGATE_CLUSTER_NAME = "service-name-2-aggr"
+        const val AGGREGATE_CLUSTER_NAME = "service-name-2-aggregate"
         const val CLUSTER_NAME_2 = "cluster-name-2"
         const val DEFAULT_SERVICE_NAME = "service-name"
         const val SERVICE_NAME_2 = "service-name-2"
@@ -252,7 +251,7 @@ class EnvoySnapshotFactoryTest {
         val cluster2 =
             createCluster(snapshotPropertiesWithWeights, serviceName = SERVICE_NAME_2, clusterName = SERVICE_NAME_2)
         val group: Group = createServicesGroup(
-            dependencies = arrayOf(cluster2.name to outgoingTimeoutPolicy(connectionIdleTimeout = 10)),
+            dependencies = arrayOf(cluster2.name to null),
             snapshotProperties = snapshotPropertiesWithWeights
         )
         val globalSnapshot = createGlobalSnapshot(cluster1, cluster2)
@@ -309,7 +308,6 @@ class EnvoySnapshotFactoryTest {
     }
 
     @Test
-    @Ignore // todo AD
     fun `should create weighted snapshot clusters for wildcard dependencies`() {
         // given
         val envoySnapshotFactory = createSnapshotFactory(snapshotPropertiesWithWeights)
@@ -318,9 +316,10 @@ class EnvoySnapshotFactoryTest {
             createCluster(snapshotPropertiesWithWeights, serviceName = SERVICE_NAME_2, clusterName = SERVICE_NAME_2)
         val wildcardTimeoutPolicy = outgoingTimeoutPolicy(connectionIdleTimeout = 12)
 
-        val group: Group = createServicesGroup(
+        val group: Group = createAllServicesGroup(
             dependencies = arrayOf("*" to wildcardTimeoutPolicy),
-            snapshotProperties = snapshotPropertiesWithWeights
+            snapshotProperties = snapshotPropertiesWithWeights,
+            defaultServiceSettings = DependencySettings(),
         )
         val globalSnapshot = createGlobalSnapshot(cluster1, cluster2)
 
@@ -334,22 +333,6 @@ class EnvoySnapshotFactoryTest {
             .containsKey(AGGREGATE_CLUSTER_NAME)
     }
 
-    @Test
-    fun `should create weighted secondary snapshot cluster with filtered endpoints`() {
-    }
-
-    @Test
-    fun `should create weighted main snapshot cluster with default endpoints`() {
-    }
-
-    @Test
-    fun `should create weighted main snapshot cluster with custom connection idle timeout`() {
-    }
-
-    @Test
-    fun `should create weighted secondary snapshot cluster with custom connection idle timeout`() {
-    }
-
     @Test
     fun `should fetch ratelimit service endpoint if there are global rate limits`() {
         // given

diff --git a/...tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/envoy/EnvoyAdmin.kt b/...tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/envoy/EnvoyAdmin.kt
@@ -69,8 +69,8 @@ class EnvoyAdmin(
         }
     }
 
-    fun configDump(): String {
-        val response = get("config_dump", mapOf("include_eds" to "on"))
+    private fun configDump(): String {
+        val response = get("config_dump")
         return response.body.use { it!!.string() }
     }
 
@@ -108,15 +108,11 @@ class EnvoyAdmin(
     private val client = OkHttpClient.Builder()
         .build()
 
-    private fun get(path: String, queryParams: Map<String, String> = mapOf()): Response {
-        val params = queryParams.entries
-            .joinToString(prefix = "?", separator = "&") {
-                "${it.key}=${it.value}"
-            }
+    private fun get(path: String): Response {
         return client.newCall(
             Request.Builder()
                 .get()
-                .url("$address/$path$params")
+                .url("$address/$path")
                 .build()
         )
             .execute().addToCloseableResponses()

diff --git a/...pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/AggregateClusterFallbackTest.kt b/...pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/AggregateClusterFallbackTest.kt
@@ -3,13 +3,15 @@ package pl.allegro.tech.servicemesh.envoycontrol.trafficsplitting
 import TrafficSplittingConstants.serviceName
 import TrafficSplittingConstants.upstreamServiceName
 import callUpstreamServiceRepeatedly
+import org.assertj.core.api.Assertions
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.RegisterExtension
-import pl.allegro.tech.servicemesh.envoycontrol.config.Xds
+import pl.allegro.tech.servicemesh.envoycontrol.assertions.untilAsserted
 import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulClusterSetup
 import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulMultiClusterExtension
 import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension
 import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlClusteredExtension
+import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoContainer
 import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension
 import verifyCallsCountCloseTo
 import verifyIsReachable
@@ -18,7 +20,6 @@ import java.time.Duration
 
 class AggregateClusterFallbackTest {
     companion object {
-        private const val numberOfCalls = 100
         private const val forceTrafficZone = "dc3"
         private val properties = mapOf(
             "envoy-control.envoy.snapshot.stateSampleDuration" to Duration.ofSeconds(0),
@@ -45,17 +46,6 @@ class AggregateClusterFallbackTest {
             )
         )
 
-        private val echo2Config = """
-            node:
-              metadata:
-                proxy_settings:
-                  outgoing:
-                    dependencies:
-                      - service: "service-1"
-        """.trimIndent()
-
-        private val config = Xds.copy(configOverride = echo2Config, serviceName = "echo2")
-
         @JvmField
         @RegisterExtension
         val consul = ConsulMultiClusterExtension()
@@ -75,7 +65,6 @@ class AggregateClusterFallbackTest {
         val envoyControl3 =
             EnvoyControlClusteredExtension(consul.serverThird, { properties }, listOf(consul))
 
-
         @JvmField
         @RegisterExtension
         val upstreamServiceDC1 = EchoServiceExtension()
@@ -132,5 +121,16 @@ class AggregateClusterFallbackTest {
         }
     }
 
+    private fun EnvoyExtension.assertInstancesUp(vararg containers: EchoContainer) {
+        untilAsserted {
+            val addresses = this.container.admin()
+                .endpointsAddress(clusterName = upstreamServiceName)
+                .map { "${it.address}:${it.portValue}" }
+            Assertions.assertThat(addresses)
+                .hasSize(containers.size)
+                .containsExactlyInAnyOrderElementsOf(containers.map { it.address() })
+        }
+    }
+
 }
 
diff --git a/.../pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt b/.../pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt
@@ -1,10 +1,12 @@
 package pl.allegro.tech.servicemesh.envoycontrol.trafficsplitting
 
+import TrafficSplittingConstants.serviceName
+import TrafficSplittingConstants.upstreamServiceName
+import callUpstreamServiceRepeatedly
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.RegisterExtension
 import pl.allegro.tech.servicemesh.envoycontrol.config.Xds
 import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulMultiClusterExtension
-import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.CallStats
 import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension
 import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlClusteredExtension
 import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension
@@ -15,12 +17,14 @@ import java.time.Duration
 
 class WeightedClustersRoutingTest {
     companion object {
-        private const val serviceName = "echo2"
-        private const val upstreamServiceName = "service-1"
-        private const val numberOfCalls = 100
         private const val forceTrafficZone = "dc2"
 
-        private val priorityProps = mapOf(
+        private val properties = mapOf(
+            "envoy-control.envoy.snapshot.stateSampleDuration" to Duration.ofSeconds(0),
+            "envoy-control.sync.enabled" to true,
+            "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.zoneName" to forceTrafficZone,
+            "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.serviceByWeightsProperties"
+                to mutableMapOf(serviceName to mutableMapOf("main" to 90, "secondary" to 10)),
             "envoy-control.envoy.snapshot.loadBalancing.priorities.zonePriorities" to mapOf(
                 "dc1" to mapOf(
                     "dc1" to 0,
@@ -33,14 +37,6 @@ class WeightedClustersRoutingTest {
             )
         )
 
-        private val properties = mapOf(
-            "envoy-control.envoy.snapshot.stateSampleDuration" to Duration.ofSeconds(0),
-            "envoy-control.sync.enabled" to true,
-            "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.zoneName" to forceTrafficZone,
-            "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.serviceByWeightsProperties"
-                to mutableMapOf(serviceName to mutableMapOf("main" to 90, "secondary" to 10))
-        )
-
         private val echo2Config = """
             node:
               metadata:
@@ -54,25 +50,17 @@ class WeightedClustersRoutingTest {
 
         @JvmField
         @RegisterExtension
-        val consulClusters = ConsulMultiClusterExtension()
+        val consul = ConsulMultiClusterExtension()
 
         @JvmField
         @RegisterExtension
         val envoyControl =
-            EnvoyControlClusteredExtension(
-                consulClusters.serverFirst,
-                { properties + priorityProps },
-                listOf(consulClusters)
-            )
+            EnvoyControlClusteredExtension(consul.serverFirst, { properties }, listOf(consul))
 
         @JvmField
         @RegisterExtension
         val envoyControl2 =
-            EnvoyControlClusteredExtension(
-                consulClusters.serverSecond,
-                { properties + priorityProps },
-                listOf(consulClusters)
-            )
+            EnvoyControlClusteredExtension(consul.serverSecond, { properties }, listOf(consul))
 
         @JvmField
         @RegisterExtension
@@ -97,32 +85,18 @@ class WeightedClustersRoutingTest {
 
     @Test
     fun `should route traffic according to weights`() {
-        consulClusters.serverFirst.operations.registerServiceWithEnvoyOnEgress(echoEnvoyDC1, name = serviceName)
+        consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(echoEnvoyDC1, name = serviceName)
 
-        consulClusters.serverFirst.operations.registerService(upstreamServiceDC1, name = upstreamServiceName)
+        consul.serverFirst.operations.registerService(upstreamServiceDC1, name = upstreamServiceName)
         echoEnvoyDC1.verifyIsReachable(upstreamServiceDC1, upstreamServiceName)
 
-        consulClusters.serverSecond.operations.registerService(upstreamServiceDC2, name = upstreamServiceName)
+        consul.serverSecond.operations.registerService(upstreamServiceDC2, name = upstreamServiceName)
         envoyDC2.verifyIsReachable(upstreamServiceDC2, upstreamServiceName)
 
         echoEnvoyDC1.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2)
             .verifyCallsCountCloseTo(upstreamServiceDC1, 90)
             .verifyCallsCountGreaterThan(upstreamServiceDC2, 1)
     }
 
-    fun EnvoyExtension.callUpstreamServiceRepeatedly(
-        vararg services: EchoServiceExtension
-    ): CallStats {
-        val stats = CallStats(services.asList())
-        this.egressOperations.callServiceRepeatedly(
-            service = upstreamServiceName,
-            stats = stats,
-            minRepeat = numberOfCalls,
-            maxRepeat = numberOfCalls,
-            repeatUntil = { true },
-            headers = mapOf()
-        )
-        return stats
-    }
 }