From a7d946e7a451401392b16d52e1071947073a72df Mon Sep 17 00:00:00 2001 From: Nastassia Dailidava <133115055+nastassia-dailidava@users.noreply.github.com> Date: Wed, 24 Apr 2024 12:09:43 +0200 Subject: [PATCH] #624 Set flat priority only for services with traffic splitting (#414) * #624 Set flat priority only for services with traffic splitting --- CHANGELOG.md | 6 + docs/configuration.md | 3 +- .../servicemesh/envoycontrol/ControlPlane.kt | 2 +- .../snapshot/EnvoySnapshotFactory.kt | 1 - .../snapshot/SnapshotProperties.kt | 1 + .../resource/clusters/EnvoyClustersFactory.kt | 14 +- .../endpoints/EnvoyEndpointsFactory.kt | 44 +++++- .../envoycontrol/EnvoySnapshotFactoryTest.kt | 15 +- .../snapshot/SnapshotUpdaterTest.kt | 2 +- .../clusters/EnvoyClustersFactoryTest.kt | 35 ++++- .../endpoints/EnvoyEndpointsFactoryTest.kt | 62 +++++++- .../envoycontrol/utils/EndpointsOperations.kt | 8 +- .../envoycontrol/utils/TestData.kt | 3 + .../LocalityWeightedLoadBalancingTest.kt | 146 ++++++++++++++++++ ...eightedLoadBalancingUnlistedServiceTest.kt | 92 +++++++++++ .../trafficsplitting/TrafficSplitting.kt | 40 +++-- .../WeightedClustersRoutingTest.kt | 137 ---------------- 17 files changed, 431 insertions(+), 180 deletions(-) create mode 100644 envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingTest.kt create mode 100644 envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingUnlistedServiceTest.kt delete mode 100644 envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index f60061aa0..fef02f9a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ Lists all changes with user impact. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). + +## [0.20.13] +### Changed +- Added setting: "zonesAllowingTrafficSplitting", so changes in a config would be made only for envoys in that zone +- Fixed setting priority for traffic splitting endpoints, they will be duplicated with higher priorities + ## [0.20.12] ### Changed - Added "trackRemaining" flag to enable possibility of tracking additional circuit breaker metrics diff --git a/docs/configuration.md b/docs/configuration.md index 39acb31e4..73b3d69e8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -138,7 +138,8 @@ Property **envoy-control.envoy.snapshot.load-balancing.policy** | load balancing policy used for clusters. [Accepted values](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#enum-config-cluster-v3-cluster-lbpolicy) | LEAST_REQUEST **envoy-control.envoy.snapshot.load-balancing.use-keys-subset-fallback-policy** | KEYS_SUBSET fallback policy is used by default when canary and service-tags are enabled. It is not supported in Envoy <= 1.12.x. Set to false for compatibility with Envoy 1.12.x | true **envoy-control.envoy.snapshot.load-balancing.traffic-splitting.zoneName** | a zone to which traffic will be routed if traffic splitting is enabled | "" -**envoy-control.envoy.snapshot.load-balancing.traffic-splitting.weights-by-service-properties.** | a map that maps service name to a map [zoneName: weight] | empty map +**envoy-control.envoy.snapshot.load-balancing.traffic-splitting.zones-allowing-traffic-splitting** | a zone from which traffic should be splitted | empty list +**envoy-control.envoy.snapshot.load-balancing.traffic-splitting.weights-by-service** | a map that maps service name to a map [zoneName: weight] | empty map ## Routing diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt index 6905f9f3a..d03592962 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt @@ -169,7 +169,7 @@ class ControlPlane private constructor( val envoySnapshotFactory = EnvoySnapshotFactory( ingressRoutesFactory = EnvoyIngressRoutesFactory(snapshotProperties, envoyHttpFilters, currentZone), egressRoutesFactory = EnvoyEgressRoutesFactory(snapshotProperties), - clustersFactory = EnvoyClustersFactory(snapshotProperties), + clustersFactory = EnvoyClustersFactory(snapshotProperties, currentZone), endpointsFactory = EnvoyEndpointsFactory( snapshotProperties, ServiceTagMetadataGenerator(snapshotProperties.routing.serviceTags), diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index df4745bc4..c56165dad 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -273,7 +273,6 @@ class EnvoySnapshotFactory( } } } - val rateLimitClusters = if (rateLimitEndpoints.isNotEmpty()) listOf(properties.rateLimit.serviceName) else emptyList() val rateLimitLoadAssignments = rateLimitClusters.mapNotNull { name -> globalSnapshot.endpoints[name] } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt index b85ffe283..9eae0d922 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt @@ -166,6 +166,7 @@ class TrafficSplittingProperties { var zoneName = "" var headerName = "" var weightsByService: Map = mapOf() + var zonesAllowingTrafficSplitting = listOf() } class ZoneWeights { diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt index 846b6381c..441fb51fb 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt @@ -52,13 +52,13 @@ import pl.allegro.tech.servicemesh.envoycontrol.snapshot.GlobalSnapshot import pl.allegro.tech.servicemesh.envoycontrol.snapshot.OAuthProvider import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.Threshold -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.TrafficSplittingProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.filters.SanUriMatcherFactory typealias EnvoyClusterConfig = io.envoyproxy.envoy.extensions.clusters.aggregate.v3.ClusterConfig class EnvoyClustersFactory( - private val properties: SnapshotProperties + private val properties: SnapshotProperties, + private val currentZone: String ) { private val httpProtocolOptions: HttpProtocolOptions = HttpProtocolOptions.newBuilder().setIdleTimeout( Durations.fromMillis(properties.egress.commonHttp.connectionIdleTimeout.toMillis()) @@ -283,14 +283,18 @@ class EnvoyClustersFactory( ): Boolean { val trafficSplitting = properties.loadBalancing.trafficSplitting val trafficSplitEnabled = trafficSplitting.weightsByService.containsKey(serviceName) - return trafficSplitEnabled && hasEndpointsInZone(clusterLoadAssignment, trafficSplitting) + val allowed = clusterLoadAssignment != null && + hasEndpointsInZone(clusterLoadAssignment, trafficSplitting.zoneName) && + trafficSplitting.zonesAllowingTrafficSplitting.contains(currentZone) + return trafficSplitEnabled && allowed } private fun hasEndpointsInZone( clusterLoadAssignment: ClusterLoadAssignment?, - trafficSplitting: TrafficSplittingProperties + zoneName: String ) = clusterLoadAssignment?.endpointsList - ?.any { e -> trafficSplitting.zoneName == e.locality.zone && e.lbEndpointsCount > 0 } ?: false + ?.any { e -> zoneName == e.locality.zone && e.lbEndpointsCount > 0 } + ?: false private fun shouldAddDynamicForwardProxyCluster(group: Group) = group.proxySettings.outgoing.getDomainPatternDependencies().isNotEmpty() diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt index 1d9e5d7db..b6ab798e3 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt @@ -34,6 +34,8 @@ class EnvoyEndpointsFactory( ) { companion object { private val logger by logger() + private const val HIGHEST_PRIORITY = 0 + private const val DEFAULT_WEIGHT = 1 } fun createLoadAssignment( @@ -84,22 +86,48 @@ class EnvoyEndpointsFactory( return if (routeSpec is WeightRouteSpecification) { ClusterLoadAssignment.newBuilder(loadAssignment) .clearEndpoints() - .addAllEndpoints(assignWeights(loadAssignment.endpointsList, routeSpec.clusterWeights)) + .addAllEndpoints( + assignWeightsAndDuplicateEndpoints( + loadAssignment.endpointsList, + routeSpec.clusterWeights + ) + ) .setClusterName(routeSpec.clusterName) .build() } else loadAssignment } - private fun assignWeights( - llbEndpointsList: List, weights: ZoneWeights + private fun assignWeightsAndDuplicateEndpoints( + llbEndpointsList: List, + weights: ZoneWeights ): List { + if (properties.loadBalancing.trafficSplitting.zonesAllowingTrafficSplitting.contains(currentZone)) { + val endpoints = llbEndpointsList + .map { + if (weights.weightByZone.containsKey(it.locality.zone)) { + LocalityLbEndpoints.newBuilder(it) + .setLoadBalancingWeight( + UInt32Value.of( + weights.weightByZone[it.locality.zone] ?: DEFAULT_WEIGHT + ) + ) + .build() + } else it + } + return overrideTrafficSplittingZoneEndpointsPriority(endpoints) + endpoints + } return llbEndpointsList + } + + private fun overrideTrafficSplittingZoneEndpointsPriority( + endpoints: List + ): List { + return endpoints + .filter { properties.loadBalancing.trafficSplitting.zoneName == it.locality.zone } .map { - if (weights.weightByZone.containsKey(it.locality.zone)) { - LocalityLbEndpoints.newBuilder(it) - .setLoadBalancingWeight(UInt32Value.of(weights.weightByZone[it.locality.zone] ?: 0)) - .build() - } else it + LocalityLbEndpoints.newBuilder(it) + .setPriority(HIGHEST_PRIORITY) + .build() } } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt index 5ba51e613..4dfc366be 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt @@ -38,9 +38,11 @@ import pl.allegro.tech.servicemesh.envoycontrol.utils.CLUSTER_NAME import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_CLUSTER_WEIGHTS import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_DISCOVERY_SERVICE_NAME import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_IDLE_TIMEOUT +import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_PRIORITY import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_SERVICE_NAME import pl.allegro.tech.servicemesh.envoycontrol.utils.EGRESS_HOST import pl.allegro.tech.servicemesh.envoycontrol.utils.EGRESS_PORT +import pl.allegro.tech.servicemesh.envoycontrol.utils.HIGHEST_PRIORITY import pl.allegro.tech.servicemesh.envoycontrol.utils.INGRESS_HOST import pl.allegro.tech.servicemesh.envoycontrol.utils.INGRESS_PORT import pl.allegro.tech.servicemesh.envoycontrol.utils.SNAPSHOT_PROPERTIES_WITH_WEIGHTS @@ -275,13 +277,18 @@ class EnvoySnapshotFactoryTest { assertThat(it.endpointsList) .anySatisfy { e -> e.locality.zone == TRAFFIC_SPLITTING_ZONE && - e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[TRAFFIC_SPLITTING_ZONE] + e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[TRAFFIC_SPLITTING_ZONE] && + e.priority == DEFAULT_PRIORITY + }.anySatisfy { e -> + e.locality.zone == TRAFFIC_SPLITTING_ZONE && + e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[TRAFFIC_SPLITTING_ZONE] && + e.priority == HIGHEST_PRIORITY } .anySatisfy { e -> e.locality.zone == CURRENT_ZONE && e.loadBalancingWeight.value == DEFAULT_CLUSTER_WEIGHTS.weightByZone[CURRENT_ZONE] } - .hasSize(2) + .hasSize(3) } } @@ -313,7 +320,7 @@ class EnvoySnapshotFactoryTest { e.locality.zone == TRAFFIC_SPLITTING_ZONE && !e.hasLoadBalancingWeight() } - .hasSize(2) + .hasSize(3) } } @@ -457,7 +464,7 @@ class EnvoySnapshotFactoryTest { CURRENT_ZONE ) val egressRoutesFactory = EnvoyEgressRoutesFactory(properties) - val clustersFactory = EnvoyClustersFactory(properties) + val clustersFactory = EnvoyClustersFactory(properties, CURRENT_ZONE) val endpointsFactory = EnvoyEndpointsFactory(properties, ServiceTagMetadataGenerator(), CURRENT_ZONE) val envoyHttpFilters = EnvoyHttpFilters.defaultFilters(properties) val listenersFactory = EnvoyListenersFactory(properties, envoyHttpFilters) diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt index 916357f72..247f879e3 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt @@ -1312,7 +1312,7 @@ class SnapshotUpdaterTest { EnvoySnapshotFactory( ingressRoutesFactory = EnvoyIngressRoutesFactory(snapshotProperties, currentZone = CURRENT_ZONE), egressRoutesFactory = EnvoyEgressRoutesFactory(snapshotProperties), - clustersFactory = EnvoyClustersFactory(snapshotProperties), + clustersFactory = EnvoyClustersFactory(snapshotProperties, CURRENT_ZONE), endpointsFactory = EnvoyEndpointsFactory( snapshotProperties, ServiceTagMetadataGenerator(snapshotProperties.routing.serviceTags), CURRENT_ZONE ), diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt index 9c82af855..8514a3841 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactoryTest.kt @@ -3,6 +3,7 @@ package pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters import io.envoyproxy.controlplane.cache.SnapshotResources import io.envoyproxy.envoy.config.cluster.v3.Cluster import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment +import io.envoyproxy.envoy.config.endpoint.v3.LocalityLbEndpoints import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import pl.allegro.tech.servicemesh.envoycontrol.groups.DependencySettings @@ -10,12 +11,14 @@ import pl.allegro.tech.servicemesh.envoycontrol.snapshot.GlobalSnapshot import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties import pl.allegro.tech.servicemesh.envoycontrol.utils.CLUSTER_NAME1 import pl.allegro.tech.servicemesh.envoycontrol.utils.CLUSTER_NAME2 +import pl.allegro.tech.servicemesh.envoycontrol.utils.CURRENT_ZONE import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_CLUSTER_WEIGHTS import pl.allegro.tech.servicemesh.envoycontrol.utils.DEFAULT_SERVICE_NAME import pl.allegro.tech.servicemesh.envoycontrol.utils.TRAFFIC_SPLITTING_ZONE import pl.allegro.tech.servicemesh.envoycontrol.utils.createAllServicesGroup import pl.allegro.tech.servicemesh.envoycontrol.utils.createCluster import pl.allegro.tech.servicemesh.envoycontrol.utils.createClusterConfigurations +import pl.allegro.tech.servicemesh.envoycontrol.utils.createEndpoints import pl.allegro.tech.servicemesh.envoycontrol.utils.createListenersConfig import pl.allegro.tech.servicemesh.envoycontrol.utils.createLoadAssignments import pl.allegro.tech.servicemesh.envoycontrol.utils.createServicesGroup @@ -23,7 +26,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.utils.createServicesGroup internal class EnvoyClustersFactoryTest { companion object { - private val factory = EnvoyClustersFactory(SnapshotProperties()) + private val factory = EnvoyClustersFactory(SnapshotProperties(), CURRENT_ZONE) private val snapshotPropertiesWithWeights = SnapshotProperties().apply { loadBalancing.trafficSplitting.weightsByService = mapOf( DEFAULT_SERVICE_NAME to DEFAULT_CLUSTER_WEIGHTS @@ -96,7 +99,7 @@ internal class EnvoyClustersFactoryTest { @Test fun `should get cluster with locality weighted config for group clusters`() { val cluster1 = createCluster(snapshotPropertiesWithWeights, CLUSTER_NAME1) - val factory = EnvoyClustersFactory(snapshotPropertiesWithWeights) + val factory = EnvoyClustersFactory(snapshotPropertiesWithWeights, CURRENT_ZONE) val result = factory.getClustersForGroup( createServicesGroup( snapshotProperties = snapshotPropertiesWithWeights, @@ -113,15 +116,39 @@ internal class EnvoyClustersFactoryTest { } } + @Test + fun `should not apply locality weighted config if there are no endpoints in the ts zone`() { + val cluster1 = createCluster(snapshotPropertiesWithWeights, CLUSTER_NAME1) + val factory = EnvoyClustersFactory(snapshotPropertiesWithWeights, CURRENT_ZONE) + val result = factory.getClustersForGroup( + createServicesGroup( + snapshotProperties = snapshotPropertiesWithWeights, + listenersConfig = createListenersConfig(snapshotPropertiesWithWeights, true), + dependencies = arrayOf(CLUSTER_NAME1 to null), + ), + createGlobalSnapshot(cluster1, endpoints = null) + ) + assertThat(result) + .anySatisfy { + assertThat(it.name).isEqualTo(CLUSTER_NAME1) + assertThat(it.edsClusterConfig).isEqualTo(cluster1.edsClusterConfig) + assertThat(it.commonLbConfig.hasLocalityWeightedLbConfig()).isFalse() + } + } + private fun createGlobalSnapshot( vararg clusters: Cluster, - securedClusters: List = clusters.asList() + securedClusters: List = clusters.asList(), + endpoints: List? = createEndpoints() ): GlobalSnapshot { + val clusterLoadAssignment = endpoints + ?.let { createLoadAssignments(clusters.toList(), endpoints) } + ?: createLoadAssignments(clusters.toList(), listOf()) return GlobalSnapshot( SnapshotResources.create(clusters.toList(), "pl/allegro/tech/servicemesh/envoycontrol/v3") .resources(), clusters.map { it.name }.toSet(), - SnapshotResources.create(createLoadAssignments(clusters.toList()), "v1").resources(), + SnapshotResources.create(clusterLoadAssignment, "v1").resources(), createClusterConfigurations(), SnapshotResources.create(securedClusters, "v3").resources() ) diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt index eb1e693e8..de151d627 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactoryTest.kt @@ -8,6 +8,7 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.MethodSource +import pl.allegro.tech.servicemesh.envoycontrol.groups.DependencySettings import pl.allegro.tech.servicemesh.envoycontrol.groups.RoutingPolicy import pl.allegro.tech.servicemesh.envoycontrol.services.ClusterState import pl.allegro.tech.servicemesh.envoycontrol.services.Locality @@ -19,6 +20,9 @@ import pl.allegro.tech.servicemesh.envoycontrol.services.ServicesState import pl.allegro.tech.servicemesh.envoycontrol.snapshot.LoadBalancingPriorityProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.LoadBalancingProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties +import pl.allegro.tech.servicemesh.envoycontrol.snapshot.TrafficSplittingProperties +import pl.allegro.tech.servicemesh.envoycontrol.snapshot.WeightRouteSpecification +import pl.allegro.tech.servicemesh.envoycontrol.snapshot.ZoneWeights import java.util.concurrent.ConcurrentHashMap import java.util.stream.Stream @@ -33,13 +37,13 @@ internal class EnvoyEndpointsFactoryTest { ), "DC2" to mapOf( "DC1" to 1, - "DC2" to 2, - "DC3" to 3 + "DC2" to 0, + "DC3" to 2 ), "DC3" to mapOf( - "DC1" to 2, - "DC2" to 3, - "DC3" to 4 + "DC1" to 1, + "DC2" to 2, + "DC3" to 0 ) ) @@ -52,6 +56,7 @@ internal class EnvoyEndpointsFactoryTest { private val serviceName = "service-one" private val defaultZone = "DC1" + private val trafficSplittingZone = "DC2" private val endpointsFactory = EnvoyEndpointsFactory( SnapshotProperties().apply { @@ -77,6 +82,31 @@ internal class EnvoyEndpointsFactoryTest { ) ) + private val defaultZoneWeights = mapOf( + "DC1" to 100, + "DC2" to 10, + "DC3" to 2 + ) + + private val snapshotPropertiesWithWeights = SnapshotProperties().apply { + loadBalancing = LoadBalancingProperties() + .apply { + priorities = LoadBalancingPriorityProperties() + .apply { zonePriorities = dcPriorityProperties } + trafficSplitting = TrafficSplittingProperties() + .apply { + zoneName = trafficSplittingZone + zonesAllowingTrafficSplitting = listOf("DC1") + weightsByService = mapOf( + serviceName to ZoneWeights() + .apply { + weightByZone = defaultZoneWeights + } + ) + } + } + } + // language=json private val globalLoadAssignmentJson = """{ "cluster_name": "lorem-service", @@ -354,6 +384,28 @@ internal class EnvoyEndpointsFactoryTest { ) } + @Test + fun `should override priority and duplicate endpoints for traffic splitting zone`() { + val envoyEndpointsFactory = EnvoyEndpointsFactory( + snapshotPropertiesWithWeights, + currentZone = "DC1" + ) + val loadAssignments = envoyEndpointsFactory.createLoadAssignment(setOf(serviceName), multiClusterStateDC1Local) + var resultLoadAssignment = envoyEndpointsFactory.assignLocalityWeights( + WeightRouteSpecification( + serviceName, listOf(), DependencySettings(), ZoneWeights().apply { weightByZone = defaultZoneWeights } + ), + loadAssignments[0] + ) + + assertThat(resultLoadAssignment.endpointsList).hasSize(dcPriorityProperties.size + 1) + assertThat(resultLoadAssignment.endpointsList) + .anySatisfy { it.hasZoneWithPriority("DC2", 1) } + .anySatisfy { it.hasZoneWithPriority("DC2", 0) } + .anySatisfy { it.hasZoneWithPriority("DC1", 0) } + .anySatisfy { it.hasZoneWithPriority("DC3", 2) } + } + private fun List.assertHasLoadAssignment(map: Map) { assertThat(this) .isNotEmpty() diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt index 38eaf4af5..12494cf45 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/EndpointsOperations.kt @@ -5,11 +5,14 @@ import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment import io.envoyproxy.envoy.config.endpoint.v3.LbEndpoint import io.envoyproxy.envoy.config.endpoint.v3.LocalityLbEndpoints -fun createLoadAssignments(clusters: List): List { +fun createLoadAssignments( + clusters: List, + endpoints: List +): List { return clusters.map { ClusterLoadAssignment.newBuilder() .setClusterName(it.name) - .addAllEndpoints(createEndpoints()) + .addAllEndpoints(endpoints) .build() } } @@ -29,5 +32,6 @@ fun createEndpoint(zone: String): LocalityLbEndpoints { .build() ) .addAllLbEndpoints(listOf(LbEndpoint.getDefaultInstance())) + .setPriority(DEFAULT_PRIORITY) .build() } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt index 4dd10f126..2ac3e9b9e 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/utils/TestData.kt @@ -15,6 +15,8 @@ const val CLUSTER_NAME1 = "cluster-1" const val CLUSTER_NAME2 = "cluster-2" const val TRAFFIC_SPLITTING_ZONE = "dc2" const val CURRENT_ZONE = "dc1" +const val DEFAULT_PRIORITY = 1 +const val HIGHEST_PRIORITY = 0 val DEFAULT_CLUSTER_WEIGHTS = zoneWeights(mapOf(CURRENT_ZONE to 60, TRAFFIC_SPLITTING_ZONE to 40)) @@ -24,6 +26,7 @@ val SNAPSHOT_PROPERTIES_WITH_WEIGHTS = SnapshotProperties().also { DEFAULT_SERVICE_NAME to DEFAULT_CLUSTER_WEIGHTS ) it.loadBalancing.trafficSplitting.zoneName = TRAFFIC_SPLITTING_ZONE + it.loadBalancing.trafficSplitting.zonesAllowingTrafficSplitting = listOf(CURRENT_ZONE) } fun zoneWeights(weightByZone: Map) = ZoneWeights().also { diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingTest.kt new file mode 100644 index 000000000..56dbdda85 --- /dev/null +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingTest.kt @@ -0,0 +1,146 @@ +package pl.allegro.tech.servicemesh.envoycontrol.trafficsplitting + +import TrafficSplitting.DEFAULT_PRIORITIES +import TrafficSplitting.FORCE_TRAFFIC_ZONE +import TrafficSplitting.SERVICE_NAME +import TrafficSplitting.UPSTREAM_SERVICE_NAME +import callUpstreamServiceRepeatedly +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.RegisterExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.Xds +import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulMultiClusterExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlClusteredExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension +import verifyCallsCountCloseTo +import verifyCallsCountEq +import verifyIsReachable +import java.time.Duration + +class LocalityWeightedLoadBalancingTest { + companion object { + private val properties = mapOf( + "envoy-control.envoy.snapshot.stateSampleDuration" to Duration.ZERO, + "envoy-control.sync.enabled" to true, + "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.zoneName" to FORCE_TRAFFIC_ZONE, + "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.zonesAllowingTrafficSplitting" to listOf("dc1"), + "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.weightsByService.$SERVICE_NAME.weightByZone.dc1" to 30, + "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.weightsByService.$SERVICE_NAME.weightByZone.dc2" to 10, + "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.weightsByService.$SERVICE_NAME.weightByZone.dc3" to 1, + "envoy-control.envoy.snapshot.loadBalancing.priorities.zonePriorities" to DEFAULT_PRIORITIES + ) + + private val echo2Config = """ + node: + metadata: + proxy_settings: + outgoing: + dependencies: + - service: "service-1" + - service: "service-2" + """.trimIndent() + + private val config = Xds.copy(configOverride = echo2Config, serviceName = "echo2") + + @JvmField + @RegisterExtension + val consul = ConsulMultiClusterExtension() + + @JvmField + @RegisterExtension + val envoyControl = + EnvoyControlClusteredExtension(consul.serverFirst, { properties }, listOf(consul)) + + @JvmField + @RegisterExtension + val envoyControl2 = + EnvoyControlClusteredExtension(consul.serverSecond, { properties }, listOf(consul)) + + @JvmField + @RegisterExtension + val envoyControl3 = + EnvoyControlClusteredExtension(consul.serverThird, { properties }, listOf(consul)) + + @JvmField + @RegisterExtension + val echoServiceDC1 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val upstreamServiceDC1 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val upstreamServiceDC2 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val upstreamServiceDC3 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val downstreamServiceEnvoy = EnvoyExtension(envoyControl, localService = echoServiceDC1, config) + + @JvmField + @RegisterExtension + val envoyDC2 = EnvoyExtension(envoyControl2) + + @JvmField + @RegisterExtension + val envoyDC3 = EnvoyExtension(envoyControl3) + } + + @Test + fun `should route traffic according to weights`() { + consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(downstreamServiceEnvoy, name = SERVICE_NAME) + + consul.serverFirst.operations.registerService(upstreamServiceDC1, name = UPSTREAM_SERVICE_NAME) + downstreamServiceEnvoy.verifyIsReachable(upstreamServiceDC1, UPSTREAM_SERVICE_NAME) + + consul.serverSecond.operations.registerService(upstreamServiceDC2, name = UPSTREAM_SERVICE_NAME) + downstreamServiceEnvoy.verifyIsReachable(upstreamServiceDC2, UPSTREAM_SERVICE_NAME) + + downstreamServiceEnvoy.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2) + .verifyCallsCountCloseTo(upstreamServiceDC1, 75) + .verifyCallsCountCloseTo(upstreamServiceDC2, 25) + .verifyCallsCountEq(upstreamServiceDC3, 0) + } + + @Test + fun `should route traffic according to weights with service tag`() { + consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(downstreamServiceEnvoy, name = SERVICE_NAME) + + consul.serverFirst.operations.registerService( + upstreamServiceDC1, + name = UPSTREAM_SERVICE_NAME, + tags = listOf("tag") + ) + downstreamServiceEnvoy.verifyIsReachable(upstreamServiceDC1, UPSTREAM_SERVICE_NAME) + consul.serverSecond.operations.registerService( + upstreamServiceDC2, + name = UPSTREAM_SERVICE_NAME, + tags = listOf("tag") + ) + downstreamServiceEnvoy.verifyIsReachable(upstreamServiceDC2, UPSTREAM_SERVICE_NAME) + downstreamServiceEnvoy.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2, tag = "tag") + .verifyCallsCountCloseTo(upstreamServiceDC1, 75) + .verifyCallsCountCloseTo(upstreamServiceDC2, 25) + .verifyCallsCountEq(upstreamServiceDC3, 0) + } + + @Test + fun `should not split traffic from unlisted zone`() { + consul.serverThird.operations.registerServiceWithEnvoyOnEgress(envoyDC3, name = "echo") + + consul.serverThird.operations.registerService(upstreamServiceDC3, name = UPSTREAM_SERVICE_NAME) + envoyDC3.verifyIsReachable(upstreamServiceDC3, UPSTREAM_SERVICE_NAME) + + consul.serverSecond.operations.registerService(upstreamServiceDC2, name = UPSTREAM_SERVICE_NAME) + envoyDC2.verifyIsReachable(upstreamServiceDC2, UPSTREAM_SERVICE_NAME) + + envoyDC3.callUpstreamServiceRepeatedly(upstreamServiceDC3, upstreamServiceDC2) + .verifyCallsCountEq(upstreamServiceDC3, 100) + .verifyCallsCountEq(upstreamServiceDC2, 0) + .verifyCallsCountEq(upstreamServiceDC1, 0) + } +} diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingUnlistedServiceTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingUnlistedServiceTest.kt new file mode 100644 index 000000000..d15cbc197 --- /dev/null +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/LocalityWeightedLoadBalancingUnlistedServiceTest.kt @@ -0,0 +1,92 @@ +package pl.allegro.tech.servicemesh.envoycontrol.trafficsplitting + +import TrafficSplitting.DEFAULT_PRIORITIES +import TrafficSplitting.FORCE_TRAFFIC_ZONE +import TrafficSplitting.SERVICE_NAME +import TrafficSplitting.UPSTREAM_SERVICE_NAME +import callUpstreamServiceRepeatedly +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.RegisterExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.Xds +import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulMultiClusterExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlClusteredExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension +import pl.allegro.tech.servicemesh.envoycontrol.trafficsplitting.LocalityWeightedLoadBalancingTest.Companion.upstreamServiceDC3 +import verifyCallsCountEq +import verifyIsReachable +import java.time.Duration + +class LocalityWeightedLoadBalancingUnlistedServiceTest { + companion object { + private val properties = mapOf( + "envoy-control.envoy.snapshot.stateSampleDuration" to Duration.ZERO, + "envoy-control.sync.enabled" to true, + "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.zoneName" to FORCE_TRAFFIC_ZONE, + "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.zonesAllowingTrafficSplitting" to listOf("dc1"), + "envoy-control.envoy.snapshot.loadBalancing.priorities.zonePriorities" to DEFAULT_PRIORITIES + ) + + private val echo2Config = """ + node: + metadata: + proxy_settings: + outgoing: + dependencies: + - service: "service-1" + - service: "service-2" + """.trimIndent() + + private val config = Xds.copy(configOverride = echo2Config, serviceName = "echo2") + + @JvmField + @RegisterExtension + val consul = ConsulMultiClusterExtension() + + @JvmField + @RegisterExtension + val envoyControl = + EnvoyControlClusteredExtension(consul.serverFirst, { properties }, listOf(consul)) + + @JvmField + @RegisterExtension + val envoyControl2 = + EnvoyControlClusteredExtension(consul.serverSecond, { properties }, listOf(consul)) + + @JvmField + @RegisterExtension + val echoServiceDC1 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val upstreamServiceDC1 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val upstreamServiceDC2 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val echoEnvoyDC1 = EnvoyExtension(envoyControl, localService = echoServiceDC1, config) + + @JvmField + @RegisterExtension + val echoEnvoyDC2 = EnvoyExtension(envoyControl2) + } + + @Test + fun `should not split traffic for not listed service`() { + consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(echoEnvoyDC1, name = SERVICE_NAME) + + consul.serverFirst.operations.registerService(upstreamServiceDC1, name = UPSTREAM_SERVICE_NAME) + echoEnvoyDC1.verifyIsReachable(upstreamServiceDC1, UPSTREAM_SERVICE_NAME) + + consul.serverSecond.operations.registerService(upstreamServiceDC2, name = UPSTREAM_SERVICE_NAME) + echoEnvoyDC2.verifyIsReachable(upstreamServiceDC2, UPSTREAM_SERVICE_NAME) + + echoEnvoyDC1.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2) + .verifyCallsCountEq(upstreamServiceDC1, 100) + .verifyCallsCountEq(upstreamServiceDC2, 0) + .verifyCallsCountEq(upstreamServiceDC3, 0) + } +} diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt index a49c7f996..cbd7da660 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt @@ -1,5 +1,5 @@ -import TrafficSplitting.deltaPercentage -import TrafficSplitting.upstreamServiceName +import TrafficSplitting.DELTA_PERCENTAGE +import TrafficSplitting.UPSTREAM_SERVICE_NAME import org.assertj.core.api.Assertions import org.assertj.core.data.Percentage import pl.allegro.tech.servicemesh.envoycontrol.assertions.isFrom @@ -10,9 +10,27 @@ import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension internal object TrafficSplitting { - const val upstreamServiceName = "service-1" - const val serviceName = "echo2" - const val deltaPercentage = 20.0 + const val UPSTREAM_SERVICE_NAME = "service-1" + const val SERVICE_NAME = "echo2" + const val DELTA_PERCENTAGE = 20.0 + const val FORCE_TRAFFIC_ZONE = "dc2" + val DEFAULT_PRIORITIES = mapOf( + "dc1" to mapOf( + "dc1" to 0, + "dc2" to 1, + "dc3" to 2, + ), + "dc2" to mapOf( + "dc1" to 1, + "dc2" to 0, + "dc3" to 2, + ), + "dc3" to mapOf( + "dc1" to 2, + "dc2" to 1, + "dc3" to 0, + ) + ) } fun EnvoyExtension.verifyIsReachable(echoServiceExtension: EchoServiceExtension, service: String) { @@ -24,12 +42,12 @@ fun EnvoyExtension.verifyIsReachable(echoServiceExtension: EchoServiceExtension, } fun CallStats.verifyCallsCountCloseTo(service: EchoServiceExtension, expectedCount: Int): CallStats { - Assertions.assertThat(this.hits(service)).isCloseTo(expectedCount, Percentage.withPercentage(deltaPercentage)) + Assertions.assertThat(this.hits(service)).isCloseTo(expectedCount, Percentage.withPercentage(DELTA_PERCENTAGE)) return this } -fun CallStats.verifyCallsCountGreaterThan(service: EchoServiceExtension, hits: Int): CallStats { - Assertions.assertThat(this.hits(service)).isGreaterThan(hits) +fun CallStats.verifyCallsCountEq(service: EchoServiceExtension, expectedCount: Int): CallStats { + Assertions.assertThat(this.hits(service)).isEqualTo(expectedCount) return this } @@ -39,7 +57,7 @@ fun EnvoyExtension.callUpstreamServiceRepeatedly( ): CallStats { val stats = CallStats(services.asList()) this.egressOperations.callServiceRepeatedly( - service = upstreamServiceName, + service = UPSTREAM_SERVICE_NAME, stats = stats, minRepeat = numberOfCalls, maxRepeat = numberOfCalls, @@ -56,12 +74,12 @@ fun EnvoyExtension.callUpstreamServiceRepeatedly( ): CallStats { val stats = CallStats(services.asList()) this.egressOperations.callServiceRepeatedly( - service = upstreamServiceName, + service = UPSTREAM_SERVICE_NAME, stats = stats, minRepeat = numberOfCalls, maxRepeat = numberOfCalls, repeatUntil = { true }, headers = tag?.let { mapOf("x-service-tag" to it) } ?: emptyMap(), - ) + ) return stats } diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt deleted file mode 100644 index c5953a0dd..000000000 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt +++ /dev/null @@ -1,137 +0,0 @@ -package pl.allegro.tech.servicemesh.envoycontrol.trafficsplitting - -import TrafficSplitting.serviceName -import TrafficSplitting.upstreamServiceName -import callUpstreamServiceRepeatedly -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.extension.RegisterExtension -import pl.allegro.tech.servicemesh.envoycontrol.config.Xds -import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulMultiClusterExtension -import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension -import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlClusteredExtension -import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension -import verifyCallsCountCloseTo -import verifyIsReachable -import java.time.Duration - -class WeightedClustersRoutingTest { - companion object { - private const val forceTrafficZone = "dc2" - - private val properties = mapOf( - "pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.endpoints.EnvoyEndpointsFactory" to "DEBUG", - "pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters.EnvoyClustersFactory" to "DEBUG", - "envoy-control.envoy.snapshot.stateSampleDuration" to Duration.ofSeconds(0), - "envoy-control.sync.enabled" to true, - "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.zoneName" to forceTrafficZone, - "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weightsByService.$serviceName.weightByZone.dc1" to 30, - "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weightsByService.$serviceName.weightByZone.dc2" to 10, - "envoy-control.envoy.snapshot.load-balancing.trafficSplitting.weightsByService.$serviceName.weightByZone.dc3" to 1, - "envoy-control.envoy.snapshot.load-balancing.priorities.zonePriorities" to mapOf( - "dc1" to mapOf( - "dc1" to 0, - "dc2" to 0, - "dc3" to 3, - ), - "dc2" to mapOf( - "dc1" to 0, - "dc2" to 0, - "dc3" to 3, - ), - "dc3" to mapOf( - "dc1" to 3, - "dc2" to 3, - "dc3" to 0, - ), - ) - ) - - private val echo2Config = """ - node: - metadata: - proxy_settings: - outgoing: - dependencies: - - service: "service-1" - """.trimIndent() - - private val config = Xds.copy(configOverride = echo2Config, serviceName = "echo2") - - @JvmField - @RegisterExtension - val consul = ConsulMultiClusterExtension() - - @JvmField - @RegisterExtension - val envoyControl = - EnvoyControlClusteredExtension(consul.serverFirst, { properties }, listOf(consul)) - - @JvmField - @RegisterExtension - val envoyControl2 = - EnvoyControlClusteredExtension(consul.serverSecond, { properties }, listOf(consul)) - - @JvmField - @RegisterExtension - val envoyControl3 = - EnvoyControlClusteredExtension(consul.serverThird, { properties }, listOf(consul)) - - @JvmField - @RegisterExtension - val echoServiceDC1 = EchoServiceExtension() - - @JvmField - @RegisterExtension - val upstreamServiceDC1 = EchoServiceExtension() - - @JvmField - @RegisterExtension - val upstreamServiceDC2 = EchoServiceExtension() - - @JvmField - @RegisterExtension - val upstreamServiceDC3 = EchoServiceExtension() - - @JvmField - @RegisterExtension - val echoEnvoyDC1 = EnvoyExtension(envoyControl, localService = echoServiceDC1, config) - @JvmField - @RegisterExtension - val echoEnvoyDC2 = EnvoyExtension(envoyControl2) - - @JvmField - @RegisterExtension - val echoEnvoyDC3 = EnvoyExtension(envoyControl3) - } - - @Test - fun `should route traffic according to weights`() { - consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(echoEnvoyDC1, name = serviceName) - - consul.serverFirst.operations.registerService(upstreamServiceDC1, name = upstreamServiceName) - echoEnvoyDC1.verifyIsReachable(upstreamServiceDC1, upstreamServiceName) - - consul.serverSecond.operations.registerService(upstreamServiceDC2, name = upstreamServiceName) - echoEnvoyDC1.verifyIsReachable(upstreamServiceDC2, upstreamServiceName) - - echoEnvoyDC1.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2) - .verifyCallsCountCloseTo(upstreamServiceDC1, 75) - .verifyCallsCountCloseTo(upstreamServiceDC2, 25) - println("snapshot: " + envoyControl.app.getGlobalSnapshot(false).toString()) - } - - @Test - fun `should route traffic according to weights with service tag`() { - consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(echoEnvoyDC1, name = serviceName) - - consul.serverFirst.operations.registerService(upstreamServiceDC1, name = upstreamServiceName, tags = listOf("tag")) - echoEnvoyDC1.verifyIsReachable(upstreamServiceDC1, upstreamServiceName) - - consul.serverSecond.operations.registerService(upstreamServiceDC2, name = upstreamServiceName, tags = listOf("tag")) - echoEnvoyDC1.verifyIsReachable(upstreamServiceDC2, upstreamServiceName) - - echoEnvoyDC1.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2, tag = "tag") - .verifyCallsCountCloseTo(upstreamServiceDC1, 75) - .verifyCallsCountCloseTo(upstreamServiceDC2, 25) - } -}