Skip to content

Commit

Permalink
Adds support for emitting TCP routes with TLS backend ports + instanc…
Browse files Browse the repository at this point in the history 
…e IDs

Enables route-integrity for tcp routes, and encrypted traffic for TCP
routes.

This reverts commit 3b3caa0.
  • Loading branch information
@geofffranks @ameowlia
geofffranks authored and ameowlia committed Aug 5, 2024
1 parent a43b37c commit b92e47a
Show file tree
Hide file tree
Showing 14 changed files with 403 additions and 739 deletions.
1 change: 1 addition & 0 deletions cmd/route-emitter/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ type RouteEmitterConfig struct {
RouteEmittingWorkers int `json:"route_emitting_workers,omitempty"`
SyncInterval durationjson.Duration `json:"sync_interval,omitempty"`
TCPRouteTTL durationjson.Duration `json:"tcp_route_ttl,omitempty"`
TCPEnableTLS bool `json:"tcp_enable_tls"`
OAuth OAuthConfig `json:"oauth"`
RoutingAPI RoutingAPIConfig `json:"routing_api"`
EnableTCPEmitter bool `json:"enable_tcp_emitter"`
Expand Down
2 changes: 2 additions & 0 deletions cmd/route-emitter/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ var _ = Describe("Config", func() {
"lock_retry_interval": "15s",
"lock_ttl": "20s",
"tcp_route_ttl": "2m",
"tcp_enable_tls": true,
"log_level": "debug",
"debug_address": "127.0.0.1:9999",
"enable_tcp_emitter": true,
Expand Down Expand Up @@ -131,6 +132,7 @@ var _ = Describe("Config", func() {
LockTTL: durationjson.Duration(20 * time.Second),
RouteEmittingWorkers: 18,
TCPRouteTTL: durationjson.Duration(2 * time.Minute),
TCPEnableTLS: true,
ReportInterval: durationjson.Duration(1 * time.Minute),
EnableTCPEmitter: true,
EnableInternalEmitter: true,
Expand Down
4 changes: 2 additions & 2 deletions cmd/route-emitter/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func main() {
bbsClient := initializeBBSClient(logger, cfg)

localMode := cfg.CellID != ""
table := routingtable.NewRoutingTable(cfg.RegisterDirectInstanceRoutes, metronClient)
table := routingtable.NewRoutingTable(cfg.RegisterDirectInstanceRoutes, cfg.TCPEnableTLS, metronClient)
natsEmitter := initializeNatsEmitter(logger, natsClient, cfg.RouteEmittingWorkers, metronClient, cfg.EnableInternalEmitter)

routeTTL := time.Duration(cfg.TCPRouteTTL)
Expand Down Expand Up @@ -124,7 +124,7 @@ func main() {

unregistrationCache := unregistration.NewCache(logger)

handler := routehandlers.NewHandler(table, natsEmitter, routingAPIEmitter, localMode, metronClient, unregistrationCache)
handler := routehandlers.NewHandler(table, natsEmitter, routingAPIEmitter, localMode, cfg.TCPEnableTLS, metronClient, unregistrationCache)

watcher := watcher.NewWatcher(
cfg.CellID,
Expand Down
8 changes: 4 additions & 4 deletions cmd/route-emitter/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -473,8 +473,8 @@ var _ = Describe("Route Emitter", func() {
cfgs = append(cfgs, func(cfg *config.RouteEmitterConfig) {
cfg.EnableTCPEmitter = true
})
expectedTcpRouteMapping = apimodels.NewTcpRouteMapping("", 5222, "some-ip", 62003, 120)
notExpectedTcpRouteMapping = apimodels.NewTcpRouteMapping("", 1883, "some-ip-1", 62003, 120)
expectedTcpRouteMapping = apimodels.NewTcpRouteMapping("", 5222, "some-ip", 62003, -1, "", nil, 120, apimodels.ModificationTag{})
notExpectedTcpRouteMapping = apimodels.NewTcpRouteMapping("", 1883, "some-ip-1", 62003, -1, "", nil, 120, apimodels.ModificationTag{})
expectedTcpRouteMapping.RouterGroupGuid = routerGUID
notExpectedTcpRouteMapping.RouterGroupGuid = routerGUID
cellID = ""
Expand Down Expand Up @@ -711,7 +711,7 @@ var _ = Describe("Route Emitter", func() {
By("unblocking the sync loop")
close(blkChannel)

expectedTcpRouteMapping = apimodels.NewTcpRouteMapping(routerGUID, 5222, "some-ip", 5222, 120)
expectedTcpRouteMapping = apimodels.NewTcpRouteMapping(routerGUID, 5222, "some-ip", 5222, -1, "", nil, 120, apimodels.ModificationTag{})

Eventually(routingAPIClient.TcpRouteMappings, 5*time.Second).Should(
ContainElement(matchTCPRouteMapping(expectedTcpRouteMapping)),
Expand Down Expand Up @@ -956,7 +956,7 @@ var _ = Describe("Route Emitter", func() {
By("unblocking the sync loop")
close(blkChannel)

expectedTcpRouteMapping = apimodels.NewTcpRouteMapping(routerGUID, 5222, "some-ip", 5222, 120)
expectedTcpRouteMapping = apimodels.NewTcpRouteMapping(routerGUID, 5222, "some-ip", 5222, -1, "", nil, 120, apimodels.ModificationTag{})

Eventually(routingAPIClient.TcpRouteMappings, 5*time.Second).Should(
ContainElement(matchTCPRouteMapping(expectedTcpRouteMapping)),
Expand Down
10 changes: 5 additions & 5 deletions emitter/routing_api_emitter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,11 @@ var _ = Describe("RoutingAPIEmitter", func() {
routingAPIEmitter = emitter.NewRoutingAPIEmitter(logger, routingApiClient, uaaTokenFetcher, ttl)

routingEvents = routingtable.TCPRouteMappings{
Registrations: []apimodels.TcpRouteMapping{apimodels.NewTcpRouteMapping("123", 61000, "some-ip-1", 62003, 0)},
Registrations: []apimodels.TcpRouteMapping{apimodels.NewTcpRouteMapping("123", 61000, "some-ip-1", 62003, 0, "", nil, 0, apimodels.ModificationTag{})},
}

expectedRoutingEvents = routingtable.TCPRouteMappings{
Registrations: []apimodels.TcpRouteMapping{apimodels.NewTcpRouteMapping("123", 61000, "some-ip-1", 62003, int(ttl))},
Registrations: []apimodels.TcpRouteMapping{apimodels.NewTcpRouteMapping("123", 61000, "some-ip-1", 62003, 0, "", nil, int(ttl), apimodels.ModificationTag{})},
}

token := &oauth2.Token{
Expand Down Expand Up @@ -100,10 +100,10 @@ var _ = Describe("RoutingAPIEmitter", func() {
Context("and there are unregistration events", func() {
BeforeEach(func() {
routingEvents = routingtable.TCPRouteMappings{
Unregistrations: []apimodels.TcpRouteMapping{apimodels.NewTcpRouteMapping("123", 61000, "some-ip-1", 62003, 0)},
Unregistrations: []apimodels.TcpRouteMapping{apimodels.NewTcpRouteMapping("123", 61000, "some-ip-1", 62003, 0, "", nil, 0, apimodels.ModificationTag{})},
}
expectedRoutingEvents = routingtable.TCPRouteMappings{
Unregistrations: []apimodels.TcpRouteMapping{apimodels.NewTcpRouteMapping("123", 61000, "some-ip-1", 62003, 60)},
Unregistrations: []apimodels.TcpRouteMapping{apimodels.NewTcpRouteMapping("123", 61000, "some-ip-1", 62003, 0, "", nil, 60, apimodels.ModificationTag{})},
}
})

Expand Down Expand Up @@ -166,7 +166,7 @@ var _ = Describe("RoutingAPIEmitter", func() {
BeforeEach(func() {
routingApiClient.DeleteTcpRouteMappingsReturns(errors.New("unauthorized"))
routingEvents = routingtable.TCPRouteMappings{
Unregistrations: []apimodels.TcpRouteMapping{apimodels.NewTcpRouteMapping("123", 61000, "some-ip-1", 62003, int(ttl))},
Unregistrations: []apimodels.TcpRouteMapping{apimodels.NewTcpRouteMapping("123", 61000, "some-ip-1", 62003, 0, "", nil, int(ttl), apimodels.ModificationTag{})},
}
})

Expand Down
7 changes: 6 additions & 1 deletion routehandlers/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type Handler struct {
natsEmitter emitter.NATSEmitter
routingAPIEmitter emitter.RoutingAPIEmitter
localMode bool
tcpTLSEnabled bool
metronClient loggingclient.IngressClient
unregistrationCache unregistration.Cache
}
Expand All @@ -38,6 +39,7 @@ func NewHandler(
natsEmitter emitter.NATSEmitter,
routingAPIEmitter emitter.RoutingAPIEmitter,
localMode bool,
tcpTLSEnabled bool,
metronClient loggingclient.IngressClient,
unregistrationCache unregistration.Cache,
) *Handler {
Expand All @@ -46,6 +48,7 @@ func NewHandler(
natsEmitter: natsEmitter,
routingAPIEmitter: routingAPIEmitter,
localMode: localMode,
tcpTLSEnabled: tcpTLSEnabled,
metronClient: metronClient,
unregistrationCache: unregistrationCache,
}
Expand Down Expand Up @@ -152,7 +155,9 @@ func (handler *Handler) Sync(
defer logger.Debug("completed")

nullLogger := lager.NewLogger("null-logger") // ignore log messsages from the routing table
newTable := routingtable.NewRoutingTable(false, handler.metronClient)
// The newTable is only used for Swap call which only replaces table entries and does not
// update config parameters
newTable := routingtable.NewRoutingTable(false, handler.tcpTLSEnabled, handler.metronClient)

for _, lrp := range desired {
newTable.SetRoutes(nullLogger, nil, lrp)
Expand Down
4 changes: 2 additions & 2 deletions routehandlers/handler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ var _ = Describe("Handler", func() {

fakeUnregistrationCache = &ufakes.FakeCache{}

routeHandler = routehandlers.NewHandler(fakeTable, natsEmitter, fakeRoutingAPIEmitter, false, fakeMetronClient, fakeUnregistrationCache)
routeHandler = routehandlers.NewHandler(fakeTable, natsEmitter, fakeRoutingAPIEmitter, false, false, fakeMetronClient, fakeUnregistrationCache)
})

Context("when an unrecognized event is received", func() {
Expand Down Expand Up @@ -1263,7 +1263,7 @@ var _ = Describe("Handler", func() {

Context("when emitting metrics in localMode", func() {
BeforeEach(func() {
routeHandler = routehandlers.NewHandler(fakeTable, natsEmitter, nil, true, fakeMetronClient, fakeUnregistrationCache)
routeHandler = routehandlers.NewHandler(fakeTable, natsEmitter, nil, true, false, fakeMetronClient, fakeUnregistrationCache)
fakeTable.HTTPAssociationsCountReturns(5)
})

Expand Down
5 changes: 3 additions & 2 deletions routehandlers/routing_api_handler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ var _ = Describe("RoutingAPIHandler", func() {
fakeRoutingAPIEmitter = new(emitterfakes.FakeRoutingAPIEmitter)
fakeMetronClient = &mfakes.FakeIngressClient{}
fakeUnregistrationCache = &ufakes.FakeCache{}
routeHandler = routehandlers.NewHandler(fakeRoutingTable, nil, fakeRoutingAPIEmitter, false, fakeMetronClient, fakeUnregistrationCache)
routeHandler = routehandlers.NewHandler(fakeRoutingTable, nil, fakeRoutingAPIEmitter, false, false, fakeMetronClient, fakeUnregistrationCache)
})

Describe("DesiredLRP Event", func() {
Expand Down Expand Up @@ -572,7 +572,7 @@ var _ = Describe("RoutingAPIHandler", func() {
}
return nil
}
routeHandler = routehandlers.NewHandler(fakeRoutingTable, nil, fakeRoutingAPIEmitter, true, fakeMetronClient, fakeUnregistrationCache)
routeHandler = routehandlers.NewHandler(fakeRoutingTable, nil, fakeRoutingAPIEmitter, true, false, fakeMetronClient, fakeUnregistrationCache)
fakeRoutingTable.TCPAssociationsCountReturns(1)
})

Expand Down Expand Up @@ -600,6 +600,7 @@ var _ = Describe("RoutingAPIHandler", func() {
RouterGroupGuid: "router-group-guid",
ExternalPort: 61000,
HostPort: 61006,
HostTLSPort: -1,
HostIP: "some-ip",
TTL: &ttl,
},
Expand Down
24 changes: 17 additions & 7 deletions routingtable/endpoint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,41 +89,51 @@ func NewEndpoint(
type ExternalEndpointInfo struct {
RouterGroupGUID string
Port uint32
TLSEnabled bool
}

func (info ExternalEndpointInfo) Hash() interface{} {
return info
}

func (info ExternalEndpointInfo) MessageFor(e Endpoint, directInstanceRoute, _ bool) (*RegistryMessage, *tcpmodels.TcpRouteMapping, *RegistryMessage) {
tlsHostPort := -1
tlsContainerPort := -1
instanceGUID := ""
if info.TLSEnabled {
tlsHostPort = int(e.TlsProxyPort)
tlsContainerPort = int(e.ContainerTlsProxyPort)
instanceGUID = e.InstanceGUID
}
mapping := tcpmodels.NewTcpRouteMapping(
info.RouterGroupGUID,
uint16(info.Port),
e.Host,
uint16(e.Port),
tlsHostPort,
instanceGUID,
nil,
0,
tcpmodels.ModificationTag{},
)
if e.IsDirectInstanceRoute(directInstanceRoute) {
mapping = tcpmodels.NewTcpRouteMapping(
info.RouterGroupGUID,
uint16(info.Port),
e.ContainerIP,
uint16(e.ContainerPort),
tlsContainerPort,
instanceGUID,
nil,
0,
tcpmodels.ModificationTag{},
)
}
return nil, &mapping, nil
}

type ExternalEndpointInfos []ExternalEndpointInfo

func NewExternalEndpointInfo(routerGroupGUID string, port uint32) ExternalEndpointInfo {
return ExternalEndpointInfo{
RouterGroupGUID: routerGroupGUID,
Port: port,
}
}

type Route struct {
Hostname string
RouteServiceUrl string
Expand Down
Loading

0 comments on commit b92e47a

Please sign in to comment.