diff --git a/amqp/downlink.go b/amqp/downlink.go index d26749ea6..1a9f187e2 100644 --- a/amqp/downlink.go +++ b/amqp/downlink.go @@ -42,7 +42,6 @@ func (s *DefaultSubscriber) SubscribeDeviceDownlink(appID, devID string, handler } handler(s, dataDown.AppID, dataDown.DevID, *dataDown) delivery.Ack(false) - break } }() diff --git a/amqp/utils_test.go b/amqp/utils_test.go index 1bf799bb8..295c57df5 100644 --- a/amqp/utils_test.go +++ b/amqp/utils_test.go @@ -6,11 +6,10 @@ package amqp import ( "testing" - "github.com/TheThingsNetwork/go-utils/log" - "github.com/TheThingsNetwork/go-utils/log/apex" + ttnlog "github.com/TheThingsNetwork/go-utils/log" tt "github.com/TheThingsNetwork/ttn/utils/testing" ) -func getLogger(t *testing.T, tag string) log.Interface { - return apex.Wrap(tt.GetLogger(t, tag)) +func getLogger(t *testing.T, tag string) ttnlog.Interface { + return tt.GetLogger(t, tag) } diff --git a/api/broker/communication_test.go b/api/broker/communication_test.go index 298ea006e..c232cc23c 100644 --- a/api/broker/communication_test.go +++ b/api/broker/communication_test.go @@ -11,7 +11,6 @@ import ( "time" "github.com/TheThingsNetwork/go-utils/log" - "github.com/TheThingsNetwork/go-utils/log/apex" "github.com/TheThingsNetwork/ttn/api" . "github.com/TheThingsNetwork/ttn/utils/testing" . "github.com/smartystreets/assertions" @@ -51,7 +50,7 @@ func TestHandlerBrokerCommunication(t *testing.T) { a := New(t) ctx := GetLogger(t, "TestHandlerBrokerCommunication") - log.Set(apex.Wrap(ctx)) + log.Set(ctx) brk := newTestBroker() rand.Seed(time.Now().UnixNano()) @@ -145,7 +144,7 @@ func TestRouterBrokerCommunication(t *testing.T) { a := New(t) ctx := GetLogger(t, "TestRouterBrokerCommunication") - log.Set(apex.Wrap(ctx)) + log.Set(ctx) brk := newTestBroker() rand.Seed(time.Now().UnixNano()) diff --git a/api/context.go b/api/context.go index c4e6b34ae..a295ba4e4 100644 --- a/api/context.go +++ b/api/context.go @@ -28,3 +28,19 @@ func IDFromContext(ctx context.Context) (token string, err error) { } return IDFromMetadata(md) } + +func LimitAndOffsetFromContext(ctx context.Context) (limit, offset int, err error) { + md, err := MetadataFromContext(ctx) + if err != nil { + return 0, 0, err + } + limit, err = LimitFromMetadata(md) + if err != nil { + return 0, 0, err + } + offset, err = OffsetFromMetadata(md) + if err != nil { + return 0, 0, err + } + return limit, offset, nil +} diff --git a/api/handler/manager_client.go b/api/handler/manager_client.go index 35fd693d3..453d1f932 100644 --- a/api/handler/manager_client.go +++ b/api/handler/manager_client.go @@ -7,6 +7,7 @@ import ( "encoding/json" "os" "os/user" + "strconv" "sync" "github.com/TheThingsNetwork/ttn/api/protocol/lorawan" @@ -60,7 +61,8 @@ func (h *ManagerClient) UpdateAccessToken(accessToken string) { h.accessToken = accessToken } -func (h *ManagerClient) getContext() context.Context { +// GetContext returns a new context with authentication +func (h *ManagerClient) GetContext() context.Context { h.RLock() defer h.RUnlock() md := metadata.Pairs( @@ -70,9 +72,22 @@ func (h *ManagerClient) getContext() context.Context { return metadata.NewContext(context.Background(), md) } +// GetContext returns a new context with authentication, plus limit and offset for pagination +func (h *ManagerClient) GetContextWithLimitAndOffset(limit, offset int) context.Context { + h.RLock() + defer h.RUnlock() + md := metadata.Pairs( + "id", h.id, + "token", h.accessToken, + "limit", strconv.Itoa(limit), + "offset", strconv.Itoa(offset), + ) + return metadata.NewContext(context.Background(), md) +} + // GetApplication retrieves an application from the Handler func (h *ManagerClient) GetApplication(appID string) (*Application, error) { - res, err := h.applicationManagerClient.GetApplication(h.getContext(), &ApplicationIdentifier{AppId: appID}) + res, err := h.applicationManagerClient.GetApplication(h.GetContext(), &ApplicationIdentifier{AppId: appID}) if err != nil { return nil, errors.Wrap(errors.FromGRPCError(err), "Could not get application from Handler") } @@ -81,25 +96,25 @@ func (h *ManagerClient) GetApplication(appID string) (*Application, error) { // SetApplication sets an application on the Handler func (h *ManagerClient) SetApplication(in *Application) error { - _, err := h.applicationManagerClient.SetApplication(h.getContext(), in) + _, err := h.applicationManagerClient.SetApplication(h.GetContext(), in) return errors.Wrap(errors.FromGRPCError(err), "Could not set application on Handler") } // RegisterApplication registers an application on the Handler func (h *ManagerClient) RegisterApplication(appID string) error { - _, err := h.applicationManagerClient.RegisterApplication(h.getContext(), &ApplicationIdentifier{AppId: appID}) + _, err := h.applicationManagerClient.RegisterApplication(h.GetContext(), &ApplicationIdentifier{AppId: appID}) return errors.Wrap(errors.FromGRPCError(err), "Could not register application on Handler") } // DeleteApplication deletes an application and all its devices from the Handler func (h *ManagerClient) DeleteApplication(appID string) error { - _, err := h.applicationManagerClient.DeleteApplication(h.getContext(), &ApplicationIdentifier{AppId: appID}) + _, err := h.applicationManagerClient.DeleteApplication(h.GetContext(), &ApplicationIdentifier{AppId: appID}) return errors.Wrap(errors.FromGRPCError(err), "Could not delete application from Handler") } // GetDevice retrieves a device from the Handler func (h *ManagerClient) GetDevice(appID string, devID string) (*Device, error) { - res, err := h.applicationManagerClient.GetDevice(h.getContext(), &DeviceIdentifier{AppId: appID, DevId: devID}) + res, err := h.applicationManagerClient.GetDevice(h.GetContext(), &DeviceIdentifier{AppId: appID, DevId: devID}) if err != nil { return nil, errors.Wrap(errors.FromGRPCError(err), "Could not get device from Handler") } @@ -108,19 +123,20 @@ func (h *ManagerClient) GetDevice(appID string, devID string) (*Device, error) { // SetDevice sets a device on the Handler func (h *ManagerClient) SetDevice(in *Device) error { - _, err := h.applicationManagerClient.SetDevice(h.getContext(), in) + _, err := h.applicationManagerClient.SetDevice(h.GetContext(), in) return errors.Wrap(errors.FromGRPCError(err), "Could not set device on Handler") } // DeleteDevice deletes a device from the Handler func (h *ManagerClient) DeleteDevice(appID string, devID string) error { - _, err := h.applicationManagerClient.DeleteDevice(h.getContext(), &DeviceIdentifier{AppId: appID, DevId: devID}) + _, err := h.applicationManagerClient.DeleteDevice(h.GetContext(), &DeviceIdentifier{AppId: appID, DevId: devID}) return errors.Wrap(errors.FromGRPCError(err), "Could not delete device from Handler") } -// GetDevicesForApplication retrieves all devices for an application from the Handler -func (h *ManagerClient) GetDevicesForApplication(appID string) (devices []*Device, err error) { - res, err := h.applicationManagerClient.GetDevicesForApplication(h.getContext(), &ApplicationIdentifier{AppId: appID}) +// GetDevicesForApplication retrieves all devices for an application from the Handler. +// Pass a limit to indicate the maximum number of results you want to receive, and the offset to indicate how many results should be skipped. +func (h *ManagerClient) GetDevicesForApplication(appID string, limit, offset int) (devices []*Device, err error) { + res, err := h.applicationManagerClient.GetDevicesForApplication(h.GetContextWithLimitAndOffset(limit, offset), &ApplicationIdentifier{AppId: appID}) if err != nil { return nil, errors.Wrap(errors.FromGRPCError(err), "Could not get devices for application from Handler") } @@ -133,7 +149,7 @@ func (h *ManagerClient) GetDevicesForApplication(appID string) (devices []*Devic // GetDevAddr requests a random device address with the given constraints func (h *ManagerClient) GetDevAddr(constraints ...string) (types.DevAddr, error) { devAddrManager := lorawan.NewDevAddrManagerClient(h.conn) - resp, err := devAddrManager.GetDevAddr(h.getContext(), &lorawan.DevAddrRequest{ + resp, err := devAddrManager.GetDevAddr(h.GetContext(), &lorawan.DevAddrRequest{ Usage: constraints, }) if err != nil { @@ -145,7 +161,7 @@ func (h *ManagerClient) GetDevAddr(constraints ...string) (types.DevAddr, error) // DryUplink transforms the uplink payload with the payload functions provided // in the app.. func (h *ManagerClient) DryUplink(payload []byte, app *Application, port uint32) (*DryUplinkResult, error) { - res, err := h.applicationManagerClient.DryUplink(h.getContext(), &DryUplinkMessage{ + res, err := h.applicationManagerClient.DryUplink(h.GetContext(), &DryUplinkMessage{ App: app, Payload: payload, Port: port, @@ -159,7 +175,7 @@ func (h *ManagerClient) DryUplink(payload []byte, app *Application, port uint32) // DryDownlinkWithPayload transforms the downlink payload with the payload functions // provided in app. func (h *ManagerClient) DryDownlinkWithPayload(payload []byte, app *Application, port uint32) (*DryDownlinkResult, error) { - res, err := h.applicationManagerClient.DryDownlink(h.getContext(), &DryDownlinkMessage{ + res, err := h.applicationManagerClient.DryDownlink(h.GetContext(), &DryDownlinkMessage{ App: app, Payload: payload, Port: port, @@ -178,7 +194,7 @@ func (h *ManagerClient) DryDownlinkWithFields(fields map[string]interface{}, app return nil, err } - res, err := h.applicationManagerClient.DryDownlink(h.getContext(), &DryDownlinkMessage{ + res, err := h.applicationManagerClient.DryDownlink(h.GetContext(), &DryDownlinkMessage{ App: app, Fields: string(marshalled), Port: port, diff --git a/api/monitor/client.go b/api/monitor/client.go index 782e9214b..7d5ea0885 100644 --- a/api/monitor/client.go +++ b/api/monitor/client.go @@ -6,12 +6,12 @@ package monitor import ( "sync" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/api" "github.com/TheThingsNetwork/ttn/api/broker" "github.com/TheThingsNetwork/ttn/api/gateway" "github.com/TheThingsNetwork/ttn/api/router" "github.com/TheThingsNetwork/ttn/utils/errors" - "github.com/apex/log" "golang.org/x/net/context" // See https://github.com/grpc/grpc-go/issues/711" "google.golang.org/grpc" "google.golang.org/grpc/metadata" @@ -22,7 +22,7 @@ const BufferSize = 10 // Client is a wrapper around MonitorClient type Client struct { - Ctx log.Interface + Ctx ttnlog.Interface client MonitorClient conn *grpc.ClientConn @@ -36,7 +36,7 @@ type Client struct { // NewClient is a wrapper for NewMonitorClient, initializes // connection to MonitorServer on monitorAddr with default gRPC options -func NewClient(ctx log.Interface, monitorAddr string) (cl *Client, err error) { +func NewClient(ctx ttnlog.Interface, monitorAddr string) (cl *Client, err error) { cl = &Client{ Ctx: ctx, addr: monitorAddr, @@ -171,7 +171,7 @@ type gatewayClient struct { client *Client - Ctx log.Interface + Ctx ttnlog.Interface id, token string @@ -242,7 +242,7 @@ type brokerClient struct { client *Client - Ctx log.Interface + Ctx ttnlog.Interface uplink struct { init sync.Once diff --git a/api/monitor/registry.go b/api/monitor/registry.go index 0ee9dc37d..f5ccc114e 100644 --- a/api/monitor/registry.go +++ b/api/monitor/registry.go @@ -6,7 +6,7 @@ package monitor import ( "sync" - "github.com/apex/log" + ttnlog "github.com/TheThingsNetwork/go-utils/log" ) // Registry encapsulates dealing with monitor servers that might be down during startup. @@ -25,7 +25,7 @@ type Registry interface { } // NewRegistry creates a monitor client registry. -func NewRegistry(ctx log.Interface) Registry { +func NewRegistry(ctx ttnlog.Interface) Registry { return ®istry{ ctx: ctx, monitorClients: make(map[string]*Client), @@ -37,12 +37,12 @@ func NewRegistry(ctx log.Interface) Registry { } type registry struct { - ctx log.Interface + ctx ttnlog.Interface monitorClients map[string]*Client brokerClients []BrokerClient gatewayClients map[string][]GatewayClient gatewayTokens map[string]string - newMonitorClient func(ctx log.Interface, addr string) (*Client, error) + newMonitorClient func(ctx ttnlog.Interface, addr string) (*Client, error) sync.RWMutex } diff --git a/api/monitor/registry_test.go b/api/monitor/registry_test.go index 5472de370..b66951eca 100644 --- a/api/monitor/registry_test.go +++ b/api/monitor/registry_test.go @@ -7,8 +7,8 @@ import ( "errors" "testing" + ttnlog "github.com/TheThingsNetwork/go-utils/log" . "github.com/TheThingsNetwork/ttn/utils/testing" - "github.com/apex/log" . "github.com/smartystreets/assertions" ) @@ -140,7 +140,7 @@ func TestGatewayClients(t *testing.T) { }) } -var returnClient = func(ctx log.Interface, addr string) (*Client, error) { +var returnClient = func(ctx ttnlog.Interface, addr string) (*Client, error) { return &Client{ Ctx: ctx, BrokerClient: &brokerClient{}, @@ -148,6 +148,6 @@ var returnClient = func(ctx log.Interface, addr string) (*Client, error) { }, nil } -var returnError = func(ctx log.Interface, addr string) (*Client, error) { +var returnError = func(ctx ttnlog.Interface, addr string) (*Client, error) { return nil, errors.New("") } diff --git a/api/router/communication_test.go b/api/router/communication_test.go index 451285e90..288496cad 100644 --- a/api/router/communication_test.go +++ b/api/router/communication_test.go @@ -11,7 +11,6 @@ import ( "time" "github.com/TheThingsNetwork/go-utils/log" - "github.com/TheThingsNetwork/go-utils/log/apex" "github.com/TheThingsNetwork/ttn/api" "github.com/TheThingsNetwork/ttn/api/gateway" "github.com/TheThingsNetwork/ttn/api/protocol" @@ -52,7 +51,7 @@ func TestRouterCommunication(t *testing.T) { a := New(t) ctx := GetLogger(t, "TestRouterCommunication") - log.Set(apex.Wrap(ctx)) + log.Set(ctx) rtr := newTestRouter() rand.Seed(time.Now().UnixNano()) diff --git a/cmd/broker.go b/cmd/broker.go index d076cc639..001fa2e7c 100644 --- a/cmd/broker.go +++ b/cmd/broker.go @@ -14,9 +14,9 @@ import ( "google.golang.org/grpc" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/core/broker" "github.com/TheThingsNetwork/ttn/core/component" - "github.com/apex/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -27,7 +27,7 @@ var brokerCmd = &cobra.Command{ Short: "The Things Network broker", Long: ``, PreRun: func(cmd *cobra.Command, args []string) { - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "Server": fmt.Sprintf("%s:%d", viper.GetString("broker.server-address"), viper.GetInt("broker.server-port")), "Announce": fmt.Sprintf("%s:%d", viper.GetString("broker.server-address-announce"), viper.GetInt("broker.server-port")), "NetworkServer": viper.GetString("broker.networkserver-address"), @@ -38,7 +38,7 @@ var brokerCmd = &cobra.Command{ ctx.Info("Starting") // Component - component, err := component.New(ctx, "broker", fmt.Sprintf("%s:%d", viper.GetString("broker.server-address-announce"), viper.GetInt("broker.server-port"))) + component, err := component.New(ttnlog.Get(), "broker", fmt.Sprintf("%s:%d", viper.GetString("broker.server-address-announce"), viper.GetInt("broker.server-port"))) if err != nil { ctx.WithError(err).Fatal("Could not initialize component") } diff --git a/cmd/discovery.go b/cmd/discovery.go index 65b5552c7..06f25f40d 100644 --- a/cmd/discovery.go +++ b/cmd/discovery.go @@ -12,12 +12,12 @@ import ( "os/signal" "syscall" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb "github.com/TheThingsNetwork/ttn/api/discovery" "github.com/TheThingsNetwork/ttn/core/component" "github.com/TheThingsNetwork/ttn/core/discovery" "github.com/TheThingsNetwork/ttn/core/discovery/announcement" "github.com/TheThingsNetwork/ttn/core/proxy" - "github.com/apex/log" "github.com/grpc-ecosystem/grpc-gateway/runtime" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -31,7 +31,7 @@ var discoveryCmd = &cobra.Command{ Short: "The Things Network discovery", Long: ``, PreRun: func(cmd *cobra.Command, args []string) { - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "Server": fmt.Sprintf("%s:%d", viper.GetString("discovery.server-address"), viper.GetInt("discovery.server-port")), "HTTP Proxy": fmt.Sprintf("%s:%d", viper.GetString("discovery.http-address"), viper.GetInt("discovery.http-port")), "Database": fmt.Sprintf("%s/%d", viper.GetString("discovery.redis-address"), viper.GetInt("discovery.redis-db")), @@ -50,7 +50,7 @@ var discoveryCmd = &cobra.Command{ connectRedis(client) // Component - component, err := component.New(ctx, "discovery", fmt.Sprintf("%s:%d", "localhost", viper.GetInt("discovery.server-port"))) + component, err := component.New(ttnlog.Get(), "discovery", fmt.Sprintf("%s:%d", "localhost", viper.GetInt("discovery.server-port"))) if err != nil { ctx.WithError(err).Fatal("Could not initialize component") } @@ -89,6 +89,7 @@ var discoveryCmd = &cobra.Command{ pb.RegisterDiscoveryHandler(netCtx, mux, proxyConn) prxy := proxy.WithLogger(mux, ctx) + prxy = proxy.WithPagination(prxy) go func() { err := http.ListenAndServe( diff --git a/cmd/handler.go b/cmd/handler.go index d6d87a59a..6ce448b31 100644 --- a/cmd/handler.go +++ b/cmd/handler.go @@ -11,12 +11,12 @@ import ( "os/signal" "syscall" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb "github.com/TheThingsNetwork/ttn/api/handler" "github.com/TheThingsNetwork/ttn/core/component" "github.com/TheThingsNetwork/ttn/core/handler" "github.com/TheThingsNetwork/ttn/core/proxy" "github.com/TheThingsNetwork/ttn/utils/parse" - "github.com/apex/log" "github.com/grpc-ecosystem/grpc-gateway/runtime" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -31,7 +31,7 @@ var handlerCmd = &cobra.Command{ Short: "The Things Network handler", Long: ``, PreRun: func(cmd *cobra.Command, args []string) { - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "Server": fmt.Sprintf("%s:%d", viper.GetString("handler.server-address"), viper.GetInt("handler.server-port")), "HTTP Proxy": fmt.Sprintf("%s:%d", viper.GetString("handler.http-address"), viper.GetInt("handler.http-port")), "Announce": fmt.Sprintf("%s:%d", viper.GetString("handler.server-address-announce"), viper.GetInt("handler.server-port")), @@ -54,7 +54,7 @@ var handlerCmd = &cobra.Command{ connectRedis(client) // Component - component, err := component.New(ctx, "handler", fmt.Sprintf("%s:%d", viper.GetString("handler.server-address-announce"), viper.GetInt("handler.server-port"))) + component, err := component.New(ttnlog.Get(), "handler", fmt.Sprintf("%s:%d", viper.GetString("handler.server-address-announce"), viper.GetInt("handler.server-port"))) if err != nil { ctx.WithError(err).Fatal("Could not initialize component") } @@ -139,6 +139,7 @@ var handlerCmd = &cobra.Command{ pb.RegisterApplicationManagerHandler(netCtx, mux, proxyConn) prxy := proxy.WithToken(mux) + prxy = proxy.WithPagination(prxy) prxy = proxy.WithLogger(prxy, ctx) go func() { diff --git a/cmd/networkserver.go b/cmd/networkserver.go index 980e224b1..48217e6c5 100644 --- a/cmd/networkserver.go +++ b/cmd/networkserver.go @@ -11,10 +11,10 @@ import ( "strings" "syscall" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/core/component" "github.com/TheThingsNetwork/ttn/core/networkserver" "github.com/TheThingsNetwork/ttn/core/types" - "github.com/apex/log" "github.com/spf13/cobra" "github.com/spf13/viper" "google.golang.org/grpc" @@ -27,7 +27,7 @@ var networkserverCmd = &cobra.Command{ Short: "The Things Network networkserver", Long: ``, PreRun: func(cmd *cobra.Command, args []string) { - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "Server": fmt.Sprintf("%s:%d", viper.GetString("networkserver.server-address"), viper.GetInt("networkserver.server-port")), "Database": fmt.Sprintf("%s/%d", viper.GetString("networkserver.redis-address"), viper.GetInt("networkserver.redis-db")), "NetID": viper.GetString("networkserver.net-id"), @@ -46,7 +46,7 @@ var networkserverCmd = &cobra.Command{ connectRedis(client) // Component - component, err := component.New(ctx, "networkserver", fmt.Sprintf("%s:%d", viper.GetString("networkserver.server-address-announce"), viper.GetInt("networkserver.server-port"))) + component, err := component.New(ttnlog.Get(), "networkserver", fmt.Sprintf("%s:%d", viper.GetString("networkserver.server-address-announce"), viper.GetInt("networkserver.server-port"))) if err != nil { ctx.WithError(err).Fatal("Could not initialize component") } diff --git a/cmd/root.go b/cmd/root.go index 4324ea7d3..2705e5e0c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -34,7 +34,7 @@ var cfgFile string var logFile *os.File -var ctx log.Interface +var ctx ttnlog.Interface // RootCmd is executed when ttn is executed without a subcommand var RootCmd = &cobra.Command{ @@ -79,15 +79,14 @@ var RootCmd = &cobra.Command{ }), logLevel)) } - ctx = &log.Logger{ - Handler: multiHandler.New(logHandlers...), - } - // Set the API/gRPC logger - ttnlog.Set(apex.Wrap(ctx)) + ctx = apex.Wrap(&log.Logger{ + Handler: multiHandler.New(logHandlers...), + }) + ttnlog.Set(ctx) grpclog.SetLogger(grpc.Wrap(ttnlog.Get())) - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "ComponentID": viper.GetString("id"), "Description": viper.GetString("description"), "Discovery Server Address": viper.GetString("discovery-address"), diff --git a/cmd/router.go b/cmd/router.go index 7e18a501e..4b1894cff 100644 --- a/cmd/router.go +++ b/cmd/router.go @@ -10,9 +10,9 @@ import ( "os/signal" "syscall" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/core/component" "github.com/TheThingsNetwork/ttn/core/router" - "github.com/apex/log" "github.com/spf13/cobra" "github.com/spf13/viper" "google.golang.org/grpc" @@ -24,7 +24,7 @@ var routerCmd = &cobra.Command{ Short: "The Things Network router", Long: ``, PreRun: func(cmd *cobra.Command, args []string) { - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "Server": fmt.Sprintf("%s:%d", viper.GetString("router.server-address"), viper.GetInt("router.server-port")), "Announce": fmt.Sprintf("%s:%d", viper.GetString("router.server-address-announce"), viper.GetInt("router.server-port")), }).Info("Initializing Router") @@ -33,7 +33,7 @@ var routerCmd = &cobra.Command{ ctx.Info("Starting") // Component - component, err := component.New(ctx, "router", fmt.Sprintf("%s:%d", viper.GetString("router.server-address-announce"), viper.GetInt("router.server-port"))) + component, err := component.New(ttnlog.Get(), "router", fmt.Sprintf("%s:%d", viper.GetString("router.server-address-announce"), viper.GetInt("router.server-port"))) if err != nil { ctx.WithError(err).Fatal("Could not initialize component") } diff --git a/cmd/version.go b/cmd/version.go index 3d4244975..a95a8ffcc 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -6,8 +6,8 @@ package cmd import ( "time" + "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/utils/version" - "github.com/apex/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) diff --git a/core/broker/activation.go b/core/broker/activation.go index 5e7b0bbf7..521e8396d 100644 --- a/core/broker/activation.go +++ b/core/broker/activation.go @@ -10,12 +10,12 @@ import ( "sync" "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb "github.com/TheThingsNetwork/ttn/api/broker" pb_discovery "github.com/TheThingsNetwork/ttn/api/discovery" pb_handler "github.com/TheThingsNetwork/ttn/api/handler" "github.com/TheThingsNetwork/ttn/api/trace" "github.com/TheThingsNetwork/ttn/utils/errors" - "github.com/apex/log" "github.com/brocaar/lorawan" ) @@ -28,7 +28,7 @@ type challengeResponseWithHandler struct { var errDuplicateActivation = errors.New("Not handling duplicate activation on this gateway") func (b *broker) HandleActivation(activation *pb.DeviceActivationRequest) (res *pb.DeviceActivationResponse, err error) { - ctx := b.Ctx.WithFields(log.Fields{ + ctx := b.Ctx.WithFields(ttnlog.Fields{ "GatewayID": activation.GatewayMetadata.GatewayId, "AppEUI": *activation.AppEui, "DevEUI": *activation.DevEui, @@ -95,7 +95,7 @@ func (b *broker) HandleActivation(activation *pb.DeviceActivationRequest) (res * return nil, errors.Wrap(errors.FromGRPCError(err), "NetworkServer refused to prepare activation") } - ctx = ctx.WithFields(log.Fields{ + ctx = ctx.WithFields(ttnlog.Fields{ "AppID": deduplicatedActivationRequest.AppId, "DevID": deduplicatedActivationRequest.DevId, }) diff --git a/core/broker/downlink.go b/core/broker/downlink.go index 9e30ad428..1e20212f8 100644 --- a/core/broker/downlink.go +++ b/core/broker/downlink.go @@ -7,10 +7,10 @@ import ( "strings" "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb "github.com/TheThingsNetwork/ttn/api/broker" "github.com/TheThingsNetwork/ttn/api/trace" "github.com/TheThingsNetwork/ttn/utils/errors" - "github.com/apex/log" ) // ByScore is used to sort a list of DownlinkOptions based on Score @@ -21,7 +21,7 @@ func (a ByScore) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ByScore) Less(i, j int) bool { return a[i].Score < a[j].Score } func (b *broker) HandleDownlink(downlink *pb.DownlinkMessage) error { - ctx := b.Ctx.WithFields(log.Fields{ + ctx := b.Ctx.WithFields(ttnlog.Fields{ "DevEUI": *downlink.DevEui, "AppEUI": *downlink.AppEui, }) @@ -33,9 +33,11 @@ func (b *broker) HandleDownlink(downlink *pb.DownlinkMessage) error { } else { ctx.WithField("Duration", time.Now().Sub(start)).Info("Handled downlink") } - for _, monitor := range b.Monitors.BrokerClients() { - ctx.Debug("Sending downlink to monitor") - go monitor.SendDownlink(downlink) + if downlink != nil { + for _, monitor := range b.Monitors.BrokerClients() { + ctx.Debug("Sending downlink to monitor") + go monitor.SendDownlink(downlink) + } } }() diff --git a/core/broker/server.go b/core/broker/server.go index 2cdaed4c7..975c4c8e8 100644 --- a/core/broker/server.go +++ b/core/broker/server.go @@ -6,7 +6,6 @@ package broker import ( "time" - "github.com/TheThingsNetwork/go-utils/log/apex" pb "github.com/TheThingsNetwork/ttn/api/broker" "github.com/TheThingsNetwork/ttn/api/ratelimit" "github.com/TheThingsNetwork/ttn/utils/errors" @@ -125,7 +124,7 @@ func (b *brokerRPC) Activate(ctx context.Context, req *pb.DeviceActivationReques func (b *broker) RegisterRPC(s *grpc.Server) { server := &brokerRPC{broker: b} - server.SetLogger(apex.Wrap(b.Ctx)) + server.SetLogger(b.Ctx) server.RouterAssociateChanFunc = server.associateRouter server.HandlerPublishChanFunc = server.getHandlerPublish server.HandlerSubscribeChanFunc = server.getHandlerSubscribe diff --git a/core/broker/uplink.go b/core/broker/uplink.go index 9c9c7e3b7..2e0dc836a 100644 --- a/core/broker/uplink.go +++ b/core/broker/uplink.go @@ -10,6 +10,7 @@ import ( "sort" "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb "github.com/TheThingsNetwork/ttn/api/broker" pb_discovery "github.com/TheThingsNetwork/ttn/api/discovery" "github.com/TheThingsNetwork/ttn/api/networkserver" @@ -18,7 +19,6 @@ import ( "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/utils/errors" "github.com/TheThingsNetwork/ttn/utils/fcnt" - "github.com/apex/log" "github.com/brocaar/lorawan" ) @@ -87,7 +87,7 @@ func (b *broker) HandleUplink(uplink *pb.UplinkMessage) (err error) { // Request devices from NS devAddr := types.DevAddr(macPayload.FHDR.DevAddr) - ctx = ctx.WithFields(log.Fields{ + ctx = ctx.WithFields(ttnlog.Fields{ "DevAddr": devAddr, "FCnt": macPayload.FHDR.FCnt, }) @@ -150,7 +150,7 @@ func (b *broker) HandleUplink(uplink *pb.UplinkMessage) (err error) { return errors.NewErrNotFound("device that validates MIC") } - ctx = ctx.WithFields(log.Fields{ + ctx = ctx.WithFields(ttnlog.Fields{ "MICChecks": micChecks, "DevEUI": device.DevEui, "AppEUI": device.AppEui, diff --git a/core/component/component.go b/core/component/component.go index 7a5fa223e..6bf94267a 100644 --- a/core/component/component.go +++ b/core/component/component.go @@ -14,10 +14,10 @@ import ( "github.com/TheThingsNetwork/go-account-lib/claims" "github.com/TheThingsNetwork/go-account-lib/tokenkey" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb_discovery "github.com/TheThingsNetwork/ttn/api/discovery" pb_monitor "github.com/TheThingsNetwork/ttn/api/monitor" "github.com/TheThingsNetwork/ttn/api/trace" - "github.com/apex/log" "github.com/spf13/viper" "golang.org/x/net/context" // See https://github.com/grpc/grpc-go/issues/711" "google.golang.org/grpc" @@ -30,7 +30,7 @@ type Component struct { Identity *pb_discovery.Announcement Discovery pb_discovery.Client Monitors pb_monitor.Registry - Ctx log.Interface + Ctx ttnlog.Interface AccessToken string privateKey *ecdsa.PrivateKey tlsConfig *tls.Config @@ -52,12 +52,12 @@ type ManagementInterface interface { } // New creates a new Component -func New(ctx log.Interface, serviceName string, announcedAddress string) (*Component, error) { +func New(ctx ttnlog.Interface, serviceName string, announcedAddress string) (*Component, error) { go func() { memstats := new(runtime.MemStats) for range time.Tick(time.Minute) { runtime.ReadMemStats(memstats) - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "Goroutines": runtime.NumGoroutine(), "Memory": float64(memstats.Alloc) / 1000000, }).Debugf("Stats") diff --git a/core/component/grpc.go b/core/component/grpc.go index 4e11488f2..93ea18cb7 100644 --- a/core/component/grpc.go +++ b/core/component/grpc.go @@ -4,86 +4,40 @@ package component import ( - "time" - + "github.com/TheThingsNetwork/go-utils/grpc/interceptor" + "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/utils/errors" - "github.com/apex/log" "github.com/mwitkow/go-grpc-middleware" "golang.org/x/net/context" // See https://github.com/grpc/grpc-go/issues/711" "google.golang.org/grpc" - "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/peer" ) func (c *Component) ServerOptions() []grpc.ServerOption { - unary := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - var peerAddr string - peer, ok := peer.FromContext(ctx) - if ok { - peerAddr = peer.Addr.String() - } - var peerID string - meta, ok := metadata.FromContext(ctx) - if ok { - id, ok := meta["id"] - if ok && len(id) > 0 { - peerID = id[0] - } - } - logCtx := c.Ctx.WithFields(log.Fields{ - "CallerID": peerID, - "CallerIP": peerAddr, - "Method": info.FullMethod, - }) - t := time.Now() + + unaryLog := interceptor.Unary(func(req interface{}, info *grpc.UnaryServerInfo) (log.Interface, string) { + return c.Ctx, "Request" + }) + + unaryErr := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { iface, err := handler(ctx, req) err = errors.BuildGRPCError(err) - logCtx = logCtx.WithField("Duration", time.Now().Sub(t)) - if grpc.Code(err) == codes.OK || grpc.Code(err) == codes.Canceled { - logCtx.Debug("Handled request") - } else { - logCtx.WithField("ErrCode", grpc.Code(err)).WithError(err).Debug("Handled request with error") - } return iface, err } - stream := func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { - var peerAddr string - peer, ok := peer.FromContext(stream.Context()) - if ok { - peerAddr = peer.Addr.String() - } - var peerID string - meta, ok := metadata.FromContext(stream.Context()) - if ok { - id, ok := meta["id"] - if ok && len(id) > 0 { - peerID = id[0] - } - } - logCtx := c.Ctx.WithFields(log.Fields{ - "CallerID": peerID, - "CallerIP": peerAddr, - "Method": info.FullMethod, - }) - t := time.Now() - logCtx.Debug("Start stream") + streamLog := interceptor.Stream(func(req interface{}, info *grpc.StreamServerInfo) (log.Interface, string) { + return c.Ctx, "Stream" + }) + + streamErr := func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { err := handler(srv, stream) err = errors.BuildGRPCError(err) - logCtx = logCtx.WithField("Duration", time.Now().Sub(t)) - if grpc.Code(err) == codes.OK || grpc.Code(err) == codes.Canceled { - logCtx.Debug("End stream") - } else { - logCtx.WithField("ErrCode", grpc.Code(err)).WithError(err).Debug("End stream with error") - } return err } opts := []grpc.ServerOption{ - grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unary)), - grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(stream)), + grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unaryErr, unaryLog)), + grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(streamErr, streamLog)), } if c.tlsConfig != nil { diff --git a/core/discovery/announcement/cache.go b/core/discovery/announcement/cache.go index c94666fc7..a6a899a0d 100644 --- a/core/discovery/announcement/cache.go +++ b/core/discovery/announcement/cache.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/TheThingsNetwork/ttn/core/storage" "github.com/TheThingsNetwork/ttn/core/types" "github.com/bluele/gcache" ) @@ -49,7 +50,7 @@ func NewCachedAnnouncementStore(store Store, options CacheOptions) Store { listCache := gcache.New(options.ListCacheSize).Expiration(options.ListCacheExpiration).LRU(). LoaderFunc(func(k interface{}) (interface{}, error) { key := k.(string) - announcements, err := store.ListService(key) + announcements, err := store.ListService(key, nil) if err != nil { return nil, err } @@ -68,12 +69,12 @@ func NewCachedAnnouncementStore(store Store, options CacheOptions) Store { } } -func (s *cachedAnnouncementStore) List() ([]*Announcement, error) { +func (s *cachedAnnouncementStore) List(opts *storage.ListOptions) ([]*Announcement, error) { // TODO: We're not using this function. Implement cache when we start using it. - return s.backingStore.List() + return s.backingStore.List(nil) } -func (s *cachedAnnouncementStore) ListService(serviceName string) ([]*Announcement, error) { +func (s *cachedAnnouncementStore) ListService(serviceName string, opts *storage.ListOptions) ([]*Announcement, error) { l, err := s.listCache.Get(serviceName) if err != nil { return nil, err diff --git a/core/discovery/announcement/cache_test.go b/core/discovery/announcement/cache_test.go index 8f20bf23e..2cece195f 100644 --- a/core/discovery/announcement/cache_test.go +++ b/core/discovery/announcement/cache_test.go @@ -122,12 +122,12 @@ func TestCachedAnnouncementStore(t *testing.T) { a.So(err, ShouldBeNil) // List - announcements, err := s.List() + announcements, err := s.List(nil) a.So(err, ShouldBeNil) a.So(announcements, ShouldHaveLength, 3) // List - announcements, err = s.ListService("router") + announcements, err = s.ListService("router", nil) a.So(err, ShouldBeNil) a.So(announcements, ShouldHaveLength, 1) diff --git a/core/discovery/announcement/store.go b/core/discovery/announcement/store.go index 9a0f89655..87be336e8 100644 --- a/core/discovery/announcement/store.go +++ b/core/discovery/announcement/store.go @@ -16,8 +16,8 @@ import ( // Store interface for Announcements type Store interface { - List() ([]*Announcement, error) - ListService(serviceName string) ([]*Announcement, error) + List(opts *storage.ListOptions) ([]*Announcement, error) + ListService(serviceName string, opts *storage.ListOptions) ([]*Announcement, error) Get(serviceName, serviceID string) (*Announcement, error) GetMetadata(serviceName, serviceID string) ([]Metadata, error) GetForAppID(appID string) (*Announcement, error) @@ -63,15 +63,15 @@ type RedisAnnouncementStore struct { // List all Announcements // The resulting Announcements do *not* include metadata -func (s *RedisAnnouncementStore) List() ([]*Announcement, error) { - announcementsI, err := s.store.List("", nil) +func (s *RedisAnnouncementStore) List(opts *storage.ListOptions) ([]*Announcement, error) { + announcementsI, err := s.store.List("", opts) if err != nil { return nil, err } - announcements := make([]*Announcement, 0, len(announcementsI)) - for _, announcementI := range announcementsI { + announcements := make([]*Announcement, len(announcementsI)) + for i, announcementI := range announcementsI { if announcement, ok := announcementI.(Announcement); ok { - announcements = append(announcements, &announcement) + announcements[i] = &announcement } } return announcements, nil @@ -79,15 +79,15 @@ func (s *RedisAnnouncementStore) List() ([]*Announcement, error) { // ListService lists all Announcements for a given service (router/broker/handler) // The resulting Announcements *do* include metadata -func (s *RedisAnnouncementStore) ListService(serviceName string) ([]*Announcement, error) { - announcementsI, err := s.store.List(serviceName+":*", nil) +func (s *RedisAnnouncementStore) ListService(serviceName string, opts *storage.ListOptions) ([]*Announcement, error) { + announcementsI, err := s.store.List(serviceName+":*", opts) if err != nil { return nil, err } - announcements := make([]*Announcement, 0, len(announcementsI)) - for _, announcementI := range announcementsI { + announcements := make([]*Announcement, len(announcementsI)) + for i, announcementI := range announcementsI { if announcement, ok := announcementI.(Announcement); ok { - announcements = append(announcements, &announcement) + announcements[i] = &announcement announcement.Metadata, err = s.GetMetadata(announcement.ServiceName, announcement.ID) if err != nil { return nil, err diff --git a/core/discovery/announcement/store_test.go b/core/discovery/announcement/store_test.go index d761978c8..b20520724 100644 --- a/core/discovery/announcement/store_test.go +++ b/core/discovery/announcement/store_test.go @@ -120,12 +120,12 @@ func TestRedisAnnouncementStore(t *testing.T) { a.So(err, ShouldBeNil) // List - announcements, err := s.List() + announcements, err := s.List(nil) a.So(err, ShouldBeNil) a.So(announcements, ShouldHaveLength, 3) // List - announcements, err = s.ListService("router") + announcements, err = s.ListService("router", nil) a.So(err, ShouldBeNil) a.So(announcements, ShouldHaveLength, 1) diff --git a/core/discovery/discovery.go b/core/discovery/discovery.go index 85fa72bd3..3e29ad56b 100644 --- a/core/discovery/discovery.go +++ b/core/discovery/discovery.go @@ -8,6 +8,7 @@ import ( pb "github.com/TheThingsNetwork/ttn/api/discovery" "github.com/TheThingsNetwork/ttn/core/component" "github.com/TheThingsNetwork/ttn/core/discovery/announcement" + "github.com/TheThingsNetwork/ttn/core/storage" "github.com/TheThingsNetwork/ttn/utils/errors" "gopkg.in/redis.v5" ) @@ -18,7 +19,7 @@ type Discovery interface { WithCache(options announcement.CacheOptions) WithMasterAuthServers(serverID ...string) Announce(announcement *pb.Announcement) error - GetAll(serviceName string) ([]*pb.Announcement, error) + GetAll(serviceName string, limit, offest int) ([]*pb.Announcement, error) Get(serviceName string, id string) (*pb.Announcement, error) AddMetadata(serviceName string, id string, metadata *pb.Metadata) error DeleteMetadata(serviceName string, id string, metadata *pb.Metadata) error @@ -93,13 +94,19 @@ func (d *discovery) Get(serviceName string, id string) (*pb.Announcement, error) return service.ToProto(), nil } -func (d *discovery) GetAll(serviceName string) ([]*pb.Announcement, error) { - services, err := d.services.ListService(serviceName) +func (d *discovery) GetAll(serviceName string, limit, offset int) ([]*pb.Announcement, error) { + services, err := d.services.ListService(serviceName, &storage.ListOptions{ + Limit: limit, + Offset: offset, + }) if err != nil { return nil, err } serviceCopies := make([]*pb.Announcement, 0, len(services)) for _, service := range services { + if service == nil { + continue + } serviceCopies = append(serviceCopies, service.ToProto()) } return serviceCopies, nil diff --git a/core/discovery/discovery_test.go b/core/discovery/discovery_test.go index 0dd80bff7..3c3c6a0c9 100644 --- a/core/discovery/discovery_test.go +++ b/core/discovery/discovery_test.go @@ -43,7 +43,7 @@ func TestDiscoveryAnnounce(t *testing.T) { err := d.Announce(broker1a) a.So(err, ShouldBeNil) - services, err := d.GetAll("broker") + services, err := d.GetAll("broker", 0, 0) a.So(err, ShouldBeNil) a.So(services, ShouldHaveLength, 1) a.So(services[0].NetAddress, ShouldEqual, "current address") @@ -51,7 +51,7 @@ func TestDiscoveryAnnounce(t *testing.T) { err = d.Announce(broker1b) a.So(err, ShouldBeNil) - services, err = d.GetAll("broker") + services, err = d.GetAll("broker", 0, 0) a.So(err, ShouldBeNil) a.So(services, ShouldHaveLength, 1) a.So(services[0].NetAddress, ShouldEqual, "updated address") @@ -59,7 +59,7 @@ func TestDiscoveryAnnounce(t *testing.T) { err = d.Announce(broker2) a.So(err, ShouldBeNil) - services, err = d.GetAll("broker") + services, err = d.GetAll("broker", 0, 0) a.So(err, ShouldBeNil) a.So(services, ShouldHaveLength, 2) @@ -81,16 +81,16 @@ func TestDiscoveryDiscover(t *testing.T) { d.Announce(&pb.Announcement{ServiceName: "broker", Id: "broker2.1"}) d.Announce(&pb.Announcement{ServiceName: "broker", Id: "broker2.2"}) - services, err := d.GetAll("random") + services, err := d.GetAll("random", 0, 0) a.So(err, ShouldBeNil) a.So(services, ShouldBeEmpty) - services, err = d.GetAll("router") + services, err = d.GetAll("router", 0, 0) a.So(err, ShouldBeNil) a.So(services, ShouldHaveLength, 1) a.So(services[0].Id, ShouldEqual, "router2.0") - services, err = d.GetAll("broker") + services, err = d.GetAll("broker", 0, 0) a.So(err, ShouldBeNil) a.So(services, ShouldHaveLength, 2) diff --git a/core/discovery/server.go b/core/discovery/server.go index c9e762eed..31ff6429c 100644 --- a/core/discovery/server.go +++ b/core/discovery/server.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/TheThingsNetwork/go-account-lib/rights" + "github.com/TheThingsNetwork/ttn/api" pb "github.com/TheThingsNetwork/ttn/api/discovery" "github.com/TheThingsNetwork/ttn/utils/errors" "github.com/golang/protobuf/ptypes/empty" @@ -76,8 +77,8 @@ func (d *discoveryServer) checkMetadataEditRights(ctx context.Context, in *pb.Me // Check claims for AppID if appID != "" { - if !claims.AppRight(appID, rights.AppSettings) { - return errPermissionDeniedf("No access to this application") + if !claims.AppRight(appID, rights.AppDelete) { + return errPermissionDeniedf(`No "%s" rights to Application "%s"`, rights.AppDelete, appID) } } return nil @@ -141,7 +142,11 @@ func (d *discoveryServer) DeleteMetadata(ctx context.Context, in *pb.MetadataReq } func (d *discoveryServer) GetAll(ctx context.Context, req *pb.GetServiceRequest) (*pb.AnnouncementsResponse, error) { - services, err := d.discovery.GetAll(req.ServiceName) + limit, offset, err := api.LimitAndOffsetFromContext(ctx) + if err != nil { + return nil, err + } + services, err := d.discovery.GetAll(req.ServiceName, limit, offset) if err != nil { return nil, err } diff --git a/core/handler/activation.go b/core/handler/activation.go index f964ae31b..a4bbe46e3 100644 --- a/core/handler/activation.go +++ b/core/handler/activation.go @@ -7,6 +7,7 @@ import ( "fmt" "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb_broker "github.com/TheThingsNetwork/ttn/api/broker" pb "github.com/TheThingsNetwork/ttn/api/handler" "github.com/TheThingsNetwork/ttn/api/trace" @@ -15,11 +16,10 @@ import ( "github.com/TheThingsNetwork/ttn/utils/errors" "github.com/TheThingsNetwork/ttn/utils/otaa" "github.com/TheThingsNetwork/ttn/utils/random" - "github.com/apex/log" "github.com/brocaar/lorawan" ) -func (h *handler) getActivationMetadata(ctx log.Interface, activation *pb_broker.DeduplicatedDeviceActivationRequest, device *device.Device) (types.Metadata, error) { +func (h *handler) getActivationMetadata(ctx ttnlog.Interface, activation *pb_broker.DeduplicatedDeviceActivationRequest, device *device.Device) (types.Metadata, error) { ttnUp := &pb_broker.DeduplicatedUplinkMessage{ ProtocolMetadata: activation.ProtocolMetadata, GatewayMetadata: activation.GatewayMetadata, @@ -79,7 +79,7 @@ func (h *handler) HandleActivation(activation *pb_broker.DeduplicatedDeviceActiv if activation.DevEui != nil { devEUI = *activation.DevEui } - ctx := h.Ctx.WithFields(log.Fields{ + ctx := h.Ctx.WithFields(ttnlog.Fields{ "DevEUI": devEUI, "AppEUI": appEUI, "AppID": appID, diff --git a/core/handler/amqp.go b/core/handler/amqp.go index 784df57d7..6ca90db1c 100644 --- a/core/handler/amqp.go +++ b/core/handler/amqp.go @@ -4,15 +4,13 @@ package handler import ( - "github.com/apex/log" - - "github.com/TheThingsNetwork/go-utils/log/apex" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/amqp" "github.com/TheThingsNetwork/ttn/core/types" ) func (h *handler) HandleAMQP(username, password, host, exchange, downlinkQueue string) error { - h.amqpClient = amqp.NewClient(apex.Wrap(h.Ctx), username, password, host) + h.amqpClient = amqp.NewClient(h.Ctx, username, password, host) err := h.amqpClient.Connect() if err != nil { @@ -55,7 +53,7 @@ func (h *handler) HandleAMQP(username, password, host, exchange, downlinkQueue s defer publisher.Close() for up := range h.amqpUp { - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "DevID": up.DevID, "AppID": up.AppID, }).Debug("Publish Uplink") diff --git a/core/handler/amqp_test.go b/core/handler/amqp_test.go index 7603faaaf..ffbab4496 100644 --- a/core/handler/amqp_test.go +++ b/core/handler/amqp_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/TheThingsNetwork/go-utils/log/apex" "github.com/TheThingsNetwork/ttn/amqp" "github.com/TheThingsNetwork/ttn/core/component" "github.com/TheThingsNetwork/ttn/core/handler/device" @@ -25,7 +24,7 @@ func TestHandleAMQP(t *testing.T) { a := New(t) var wg WaitGroup - c := amqp.NewClient(apex.Wrap(GetLogger(t, "TestHandleAMQP")), "guest", "guest", host) + c := amqp.NewClient(GetLogger(t, "TestHandleAMQP"), "guest", "guest", host) err := c.Connect() a.So(err, ShouldBeNil) defer c.Disconnect() diff --git a/core/handler/application/store.go b/core/handler/application/store.go index e94031772..ffb786ed2 100644 --- a/core/handler/application/store.go +++ b/core/handler/application/store.go @@ -13,7 +13,7 @@ import ( // Store interface for Applications type Store interface { - List() ([]*Application, error) + List(opts *storage.ListOptions) ([]*Application, error) Get(appID string) (*Application, error) Set(new *Application, properties ...string) (err error) Delete(appID string) error @@ -42,15 +42,15 @@ type RedisApplicationStore struct { } // List all Applications -func (s *RedisApplicationStore) List() ([]*Application, error) { - applicationsI, err := s.store.List("", nil) +func (s *RedisApplicationStore) List(opts *storage.ListOptions) ([]*Application, error) { + applicationsI, err := s.store.List("", opts) if err != nil { return nil, err } - applications := make([]*Application, 0, len(applicationsI)) - for _, applicationI := range applicationsI { + applications := make([]*Application, len(applicationsI)) + for i, applicationI := range applicationsI { if application, ok := applicationI.(Application); ok { - applications = append(applications, &application) + applications[i] = &application } } return applications, nil diff --git a/core/handler/application/store_test.go b/core/handler/application/store_test.go index 5366b4fb8..e7e215ca1 100644 --- a/core/handler/application/store_test.go +++ b/core/handler/application/store_test.go @@ -56,7 +56,7 @@ func TestApplicationStore(t *testing.T) { a.So(app.Encoder, ShouldEqual, "new encoder") // List - apps, err := s.List() + apps, err := s.List(nil) a.So(err, ShouldBeNil) a.So(apps, ShouldHaveLength, 1) diff --git a/core/handler/convert_fields.go b/core/handler/convert_fields.go index 9b80e0200..18cbe526f 100644 --- a/core/handler/convert_fields.go +++ b/core/handler/convert_fields.go @@ -8,16 +8,16 @@ import ( "reflect" "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb_broker "github.com/TheThingsNetwork/ttn/api/broker" "github.com/TheThingsNetwork/ttn/core/handler/device" "github.com/TheThingsNetwork/ttn/core/handler/functions" "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/utils/errors" - "github.com/apex/log" ) // ConvertFieldsUp converts the payload to fields using payload functions -func (h *handler) ConvertFieldsUp(ctx log.Interface, ttnUp *pb_broker.DeduplicatedUplinkMessage, appUp *types.UplinkMessage, _ *device.Device) error { +func (h *handler) ConvertFieldsUp(ctx ttnlog.Interface, ttnUp *pb_broker.DeduplicatedUplinkMessage, appUp *types.UplinkMessage, _ *device.Device) error { // Find Application app, err := h.applications.Get(ttnUp.AppId) if err != nil { @@ -285,7 +285,7 @@ func (f *DownlinkFunctions) Process(payload map[string]interface{}, port uint8) } // ConvertFieldsDown converts the fields into a payload -func (h *handler) ConvertFieldsDown(ctx log.Interface, appDown *types.DownlinkMessage, ttnDown *pb_broker.DownlinkMessage, _ *device.Device) error { +func (h *handler) ConvertFieldsDown(ctx ttnlog.Interface, appDown *types.DownlinkMessage, ttnDown *pb_broker.DownlinkMessage, _ *device.Device) error { if appDown.PayloadFields == nil || len(appDown.PayloadFields) == 0 { return nil } diff --git a/core/handler/convert_lorawan.go b/core/handler/convert_lorawan.go index 93d5f3e68..33ee8e24f 100644 --- a/core/handler/convert_lorawan.go +++ b/core/handler/convert_lorawan.go @@ -4,17 +4,17 @@ package handler import ( + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb_broker "github.com/TheThingsNetwork/ttn/api/broker" "github.com/TheThingsNetwork/ttn/api/trace" "github.com/TheThingsNetwork/ttn/core/handler/device" "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/utils/errors" "github.com/TheThingsNetwork/ttn/utils/pointer" - "github.com/apex/log" "github.com/brocaar/lorawan" ) -func (h *handler) ConvertFromLoRaWAN(ctx log.Interface, ttnUp *pb_broker.DeduplicatedUplinkMessage, appUp *types.UplinkMessage, dev *device.Device) error { +func (h *handler) ConvertFromLoRaWAN(ctx ttnlog.Interface, ttnUp *pb_broker.DeduplicatedUplinkMessage, appUp *types.UplinkMessage, dev *device.Device) error { // Check for LoRaWAN if lorawan := ttnUp.ProtocolMetadata.GetLorawan(); lorawan == nil { return errors.NewErrInvalidArgument("Uplink", "does not contain LoRaWAN metadata") @@ -75,7 +75,7 @@ func (h *handler) ConvertFromLoRaWAN(ctx log.Interface, ttnUp *pb_broker.Dedupli return nil } -func (h *handler) ConvertToLoRaWAN(ctx log.Interface, appDown *types.DownlinkMessage, ttnDown *pb_broker.DownlinkMessage, dev *device.Device) error { +func (h *handler) ConvertToLoRaWAN(ctx ttnlog.Interface, appDown *types.DownlinkMessage, ttnDown *pb_broker.DownlinkMessage, dev *device.Device) error { // LoRaWAN: Unmarshal Downlink var phyPayload lorawan.PHYPayload err := phyPayload.UnmarshalBinary(ttnDown.Payload) diff --git a/core/handler/convert_metadata.go b/core/handler/convert_metadata.go index 2246af0eb..6dd27784f 100644 --- a/core/handler/convert_metadata.go +++ b/core/handler/convert_metadata.go @@ -4,14 +4,14 @@ package handler import ( + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb_broker "github.com/TheThingsNetwork/ttn/api/broker" "github.com/TheThingsNetwork/ttn/core/handler/device" "github.com/TheThingsNetwork/ttn/core/types" - "github.com/apex/log" ) // ConvertMetadata converts the protobuf matadata to application metadata -func (h *handler) ConvertMetadata(ctx log.Interface, ttnUp *pb_broker.DeduplicatedUplinkMessage, appUp *types.UplinkMessage, dev *device.Device) error { +func (h *handler) ConvertMetadata(ctx ttnlog.Interface, ttnUp *pb_broker.DeduplicatedUplinkMessage, appUp *types.UplinkMessage, dev *device.Device) error { ctx = ctx.WithField("NumGateways", len(ttnUp.GatewayMetadata)) // Transform Metadata diff --git a/core/handler/device/store.go b/core/handler/device/store.go index cba8aa1f2..657cc645e 100644 --- a/core/handler/device/store.go +++ b/core/handler/device/store.go @@ -14,8 +14,8 @@ import ( // Store interface for Devices type Store interface { - List() ([]*Device, error) - ListForApp(appID string) ([]*Device, error) + List(opts *storage.ListOptions) ([]*Device, error) + ListForApp(appID string, opts *storage.ListOptions) ([]*Device, error) Get(appID, devID string) (*Device, error) Set(new *Device, properties ...string) (err error) Delete(appID, devID string) error @@ -43,30 +43,30 @@ type RedisDeviceStore struct { } // List all Devices -func (s *RedisDeviceStore) List() ([]*Device, error) { - devicesI, err := s.store.List("", nil) +func (s *RedisDeviceStore) List(opts *storage.ListOptions) ([]*Device, error) { + devicesI, err := s.store.List("", opts) if err != nil { return nil, err } - devices := make([]*Device, 0, len(devicesI)) - for _, deviceI := range devicesI { + devices := make([]*Device, len(devicesI)) + for i, deviceI := range devicesI { if device, ok := deviceI.(Device); ok { - devices = append(devices, &device) + devices[i] = &device } } return devices, nil } // ListForApp lists all devices for a specific Application -func (s *RedisDeviceStore) ListForApp(appID string) ([]*Device, error) { - devicesI, err := s.store.List(fmt.Sprintf("%s:*", appID), nil) +func (s *RedisDeviceStore) ListForApp(appID string, opts *storage.ListOptions) ([]*Device, error) { + devicesI, err := s.store.List(fmt.Sprintf("%s:*", appID), opts) if err != nil { return nil, err } - devices := make([]*Device, 0, len(devicesI)) - for _, deviceI := range devicesI { + devices := make([]*Device, len(devicesI)) + for i, deviceI := range devicesI { if device, ok := deviceI.(Device); ok { - devices = append(devices, &device) + devices[i] = &device } } return devices, nil diff --git a/core/handler/device/store_test.go b/core/handler/device/store_test.go index 2d66efc8d..67066b982 100644 --- a/core/handler/device/store_test.go +++ b/core/handler/device/store_test.go @@ -23,7 +23,7 @@ func TestDeviceStore(t *testing.T) { a.So(err, ShouldNotBeNil) a.So(dev, ShouldBeNil) - devs, err := s.ListForApp("AppID-1") + devs, err := s.ListForApp("AppID-1", nil) a.So(err, ShouldBeNil) a.So(devs, ShouldHaveLength, 0) @@ -46,7 +46,7 @@ func TestDeviceStore(t *testing.T) { a.So(err, ShouldBeNil) a.So(dev, ShouldNotBeNil) - devs, err = s.ListForApp("AppID-1") + devs, err = s.ListForApp("AppID-1", nil) a.So(err, ShouldBeNil) a.So(devs, ShouldHaveLength, 1) @@ -81,7 +81,7 @@ func TestDeviceStore(t *testing.T) { }() // List - devices, err := s.List() + devices, err := s.List(nil) a.So(err, ShouldBeNil) a.So(devices, ShouldHaveLength, 2) @@ -94,7 +94,7 @@ func TestDeviceStore(t *testing.T) { a.So(err, ShouldNotBeNil) a.So(dev, ShouldBeNil) - devs, err = s.ListForApp("AppID-1") + devs, err = s.ListForApp("AppID-1", nil) a.So(err, ShouldBeNil) a.So(devs, ShouldHaveLength, 1) diff --git a/core/handler/downlink.go b/core/handler/downlink.go index 9df03e33c..e69573761 100644 --- a/core/handler/downlink.go +++ b/core/handler/downlink.go @@ -6,16 +6,16 @@ package handler import ( "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb_broker "github.com/TheThingsNetwork/ttn/api/broker" "github.com/TheThingsNetwork/ttn/api/trace" "github.com/TheThingsNetwork/ttn/core/types" - "github.com/apex/log" ) func (h *handler) EnqueueDownlink(appDownlink *types.DownlinkMessage) (err error) { appID, devID := appDownlink.AppID, appDownlink.DevID - ctx := h.Ctx.WithFields(log.Fields{ + ctx := h.Ctx.WithFields(ttnlog.Fields{ "AppID": appID, "DevID": devID, }) @@ -56,7 +56,7 @@ func (h *handler) EnqueueDownlink(appDownlink *types.DownlinkMessage) (err error func (h *handler) HandleDownlink(appDownlink *types.DownlinkMessage, downlink *pb_broker.DownlinkMessage) error { appID, devID := appDownlink.AppID, appDownlink.DevID - ctx := h.Ctx.WithFields(log.Fields{ + ctx := h.Ctx.WithFields(ttnlog.Fields{ "AppID": appID, "DevID": devID, "AppEUI": downlink.AppEui, diff --git a/core/handler/dry_run_test.go b/core/handler/dry_run_test.go index 0045fc012..270832dca 100644 --- a/core/handler/dry_run_test.go +++ b/core/handler/dry_run_test.go @@ -8,6 +8,7 @@ import ( pb "github.com/TheThingsNetwork/ttn/api/handler" "github.com/TheThingsNetwork/ttn/core/handler/application" + "github.com/TheThingsNetwork/ttn/core/storage" . "github.com/TheThingsNetwork/ttn/utils/testing" . "github.com/smartystreets/assertions" "golang.org/x/net/context" // See https://github.com/grpc/grpc-go/issues/711" @@ -40,9 +41,9 @@ func (s *countingStore) Count(name string) int { return val } -func (s *countingStore) List() ([]*application.Application, error) { +func (s *countingStore) List(opts *storage.ListOptions) ([]*application.Application, error) { s.inc("list") - return s.store.List() + return s.store.List(opts) } func (s *countingStore) Get(appID string) (*application.Application, error) { diff --git a/core/handler/functions/functions.go b/core/handler/functions/functions.go index e63a19bec..587cfac65 100644 --- a/core/handler/functions/functions.go +++ b/core/handler/functions/functions.go @@ -13,7 +13,7 @@ import ( var errTimeOutExceeded = errors.NewErrInternal("Code has been running to long") -func RunCode(name, code string, env map[string]interface{}, timeout time.Duration, logger Logger) (otto.Value, error) { +func RunCode(name, code string, env map[string]interface{}, timeout time.Duration, logger Logger) (val otto.Value, err error) { vm := otto.New() // load the environment @@ -32,22 +32,18 @@ func RunCode(name, code string, env map[string]interface{}, timeout time.Duratio }) vm.Run("console.log = __log") - var value otto.Value - var err error - start := time.Now() defer func() { duration := time.Since(start) if caught := recover(); caught != nil { + val = otto.Value{} if caught == errTimeOutExceeded { - value = otto.Value{} err = errors.NewErrInternal(fmt.Sprintf("Interrupted javascript execution after %v", duration)) return + } else { + err = errors.NewErrInternal(fmt.Sprintf("Fatal error in payload function: %s", caught)) } - // if this is not the our timeout interrupt, raise the panic again - // so someone else can handle it - panic(caught) } }() @@ -59,7 +55,6 @@ func RunCode(name, code string, env map[string]interface{}, timeout time.Duratio panic(errTimeOutExceeded) } }() - val, err := vm.Run(code) - return val, err + return vm.Run(code) } diff --git a/core/handler/manager_server.go b/core/handler/manager_server.go index f51a5417c..1c61f39e9 100644 --- a/core/handler/manager_server.go +++ b/core/handler/manager_server.go @@ -5,10 +5,12 @@ package handler import ( "fmt" + "strconv" "time" "github.com/TheThingsNetwork/go-account-lib/claims" "github.com/TheThingsNetwork/go-account-lib/rights" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/api" pb_broker "github.com/TheThingsNetwork/ttn/api/broker" pb "github.com/TheThingsNetwork/ttn/api/handler" @@ -16,9 +18,9 @@ import ( "github.com/TheThingsNetwork/ttn/api/ratelimit" "github.com/TheThingsNetwork/ttn/core/handler/application" "github.com/TheThingsNetwork/ttn/core/handler/device" + "github.com/TheThingsNetwork/ttn/core/storage" "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/utils/errors" - "github.com/apex/log" "github.com/golang/protobuf/ptypes/empty" "golang.org/x/net/context" // See https://github.com/grpc/grpc-go/issues/711" "google.golang.org/grpc" @@ -34,6 +36,13 @@ type handlerManager struct { clientRate *ratelimit.Registry } +func checkAppRights(claims *claims.Claims, appID string, right rights.Right) error { + if !claims.AppRight(appID, right) { + return errors.NewErrPermissionDenied(fmt.Sprintf(`No "%s" rights to Application "%s"`, right, appID)) + } + return nil +} + func (h *handlerManager) validateTTNAuthAppContext(ctx context.Context, appID string) (context.Context, *claims.Claims, error) { md, err := api.MetadataFromContext(ctx) if err != nil { @@ -75,8 +84,9 @@ func (h *handlerManager) GetDevice(ctx context.Context, in *pb.DeviceIdentifier) if err != nil { return nil, err } - if !claims.AppRight(in.AppId, rights.Devices) { - return nil, errors.NewErrPermissionDenied(fmt.Sprintf(`No "devices" rights to application "%s"`, in.AppId)) + err = checkAppRights(claims, in.AppId, rights.Devices) + if err != nil { + return nil, err } if _, err := h.handler.applications.Get(in.AppId); err != nil { @@ -115,7 +125,7 @@ func (h *handlerManager) GetDevice(ctx context.Context, in *pb.DeviceIdentifier) }) if errors.GetErrType(errors.FromGRPCError(err)) == errors.NotFound { // Re-register the device in the Broker (NetworkServer) - h.handler.Ctx.WithFields(log.Fields{ + h.handler.Ctx.WithFields(ttnlog.Fields{ "AppID": dev.AppID, "DevID": dev.DevID, "AppEUI": dev.AppEUI, @@ -146,8 +156,9 @@ func (h *handlerManager) SetDevice(ctx context.Context, in *pb.Device) (*empty.E if err != nil { return nil, err } - if !claims.AppRight(in.AppId, rights.Devices) { - return nil, errors.NewErrPermissionDenied(fmt.Sprintf(`No "devices" rights to application "%s"`, in.AppId)) + err = checkAppRights(claims, in.AppId, rights.Devices) + if err != nil { + return nil, err } if _, err := h.handler.applications.Get(in.AppId); err != nil { @@ -180,7 +191,7 @@ func (h *handlerManager) SetDevice(ctx context.Context, in *pb.Device) (*empty.E dev.StartUpdate() } else { eventType = types.CreateEvent - existingDevices, err := h.handler.devices.ListForApp(in.AppId) + existingDevices, err := h.handler.devices.ListForApp(in.AppId, nil) if err != nil { return nil, err } @@ -261,8 +272,9 @@ func (h *handlerManager) DeleteDevice(ctx context.Context, in *pb.DeviceIdentifi if err != nil { return nil, err } - if !claims.AppRight(in.AppId, rights.Devices) { - return nil, errors.NewErrPermissionDenied(fmt.Sprintf(`No "devices" rights to application "%s"`, in.AppId)) + err = checkAppRights(claims, in.AppId, rights.Devices) + if err != nil { + return nil, err } if _, err := h.handler.applications.Get(in.AppId); err != nil { @@ -297,20 +309,30 @@ func (h *handlerManager) GetDevicesForApplication(ctx context.Context, in *pb.Ap if err != nil { return nil, err } - if !claims.AppRight(in.AppId, rights.Devices) { - return nil, errors.NewErrPermissionDenied(fmt.Sprintf(`No "devices" rights to application "%s"`, in.AppId)) + err = checkAppRights(claims, in.AppId, rights.Devices) + if err != nil { + return nil, err } if _, err := h.handler.applications.Get(in.AppId); err != nil { return nil, errors.Wrap(err, "Application not registered to this Handler") } - devices, err := h.handler.devices.ListForApp(in.AppId) + limit, offset, err := api.LimitAndOffsetFromContext(ctx) + if err != nil { + return nil, err + } + + opts := &storage.ListOptions{Limit: limit, Offset: offset} + devices, err := h.handler.devices.ListForApp(in.AppId, opts) if err != nil { return nil, err } res := &pb.DeviceList{Devices: []*pb.Device{}} for _, dev := range devices { + if dev == nil { + continue + } res.Devices = append(res.Devices, &pb.Device{ AppId: dev.AppID, DevId: dev.DevID, @@ -329,6 +351,14 @@ func (h *handlerManager) GetDevicesForApplication(ctx context.Context, in *pb.Ap Altitude: dev.Altitude, }) } + + total, selected := opts.GetTotalAndSelected() + header := metadata.Pairs( + "total", strconv.Itoa(total), + "selected", strconv.Itoa(selected), + ) + grpc.SendHeader(ctx, header) + return res, nil } @@ -340,8 +370,9 @@ func (h *handlerManager) GetApplication(ctx context.Context, in *pb.ApplicationI if err != nil { return nil, err } - if !claims.AppRight(in.AppId, rights.AppSettings) { - return nil, errors.NewErrPermissionDenied(`No "settings" rights to application`) + err = checkAppRights(claims, in.AppId, rights.AppSettings) + if err != nil { + return nil, err } app, err := h.handler.applications.Get(in.AppId) if err != nil { @@ -365,8 +396,9 @@ func (h *handlerManager) RegisterApplication(ctx context.Context, in *pb.Applica if err != nil { return nil, err } - if !claims.AppRight(in.AppId, rights.AppSettings) { - return nil, errors.NewErrPermissionDenied(`No "settings" rights to application`) + err = checkAppRights(claims, in.AppId, rights.AppSettings) + if err != nil { + return nil, err } app, err := h.handler.applications.Get(in.AppId) if err != nil && errors.GetErrType(err) != errors.NotFound { @@ -409,8 +441,9 @@ func (h *handlerManager) SetApplication(ctx context.Context, in *pb.Application) if err != nil { return nil, err } - if !claims.AppRight(in.AppId, rights.AppSettings) { - return nil, errors.NewErrPermissionDenied(`No "settings" rights to application`) + err = checkAppRights(claims, in.AppId, rights.AppSettings) + if err != nil { + return nil, err } app, err := h.handler.applications.Get(in.AppId) if err != nil { @@ -440,16 +473,18 @@ func (h *handlerManager) DeleteApplication(ctx context.Context, in *pb.Applicati if err != nil { return nil, err } - if !claims.AppRight(in.AppId, rights.AppSettings) { - return nil, errors.NewErrPermissionDenied(`No "settings" rights to application`) + err = checkAppRights(claims, in.AppId, rights.AppDelete) + if err != nil { + return nil, err } + _, err = h.handler.applications.Get(in.AppId) if err != nil { return nil, err } // Get and delete all devices for this application - devices, err := h.handler.devices.ListForApp(in.AppId) + devices, err := h.handler.devices.ListForApp(in.AppId, nil) if err != nil { return nil, err } diff --git a/core/handler/mqtt.go b/core/handler/mqtt.go index 2d6dc7cf0..a0df0fabd 100644 --- a/core/handler/mqtt.go +++ b/core/handler/mqtt.go @@ -6,10 +6,9 @@ package handler import ( "time" - "github.com/TheThingsNetwork/go-utils/log/apex" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/mqtt" - "github.com/apex/log" ) // MQTTTimeout indicates how long we should wait for an MQTT publish @@ -19,7 +18,7 @@ var MQTTTimeout = 2 * time.Second var MQTTBufferSize = 10 func (h *handler) HandleMQTT(username, password string, mqttBrokers ...string) error { - h.mqttClient = mqtt.NewClient(apex.Wrap(h.Ctx), "ttnhdl", username, password, mqttBrokers...) + h.mqttClient = mqtt.NewClient(h.Ctx, "ttnhdl", username, password, mqttBrokers...) err := h.mqttClient.Connect() if err != nil { @@ -44,7 +43,7 @@ func (h *handler) HandleMQTT(username, password string, mqttBrokers ...string) e go func() { for up := range h.mqttUp { - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "DevID": up.DevID, "AppID": up.AppID, }).Debug("Publish Uplink") @@ -75,7 +74,7 @@ func (h *handler) HandleMQTT(username, password string, mqttBrokers ...string) e go func() { for event := range h.mqttEvent { - h.Ctx.WithFields(log.Fields{ + h.Ctx.WithFields(ttnlog.Fields{ "DevID": event.DevID, "AppID": event.AppID, "Event": event.Event, diff --git a/core/handler/mqtt_test.go b/core/handler/mqtt_test.go index b54f3e60c..d53dce385 100644 --- a/core/handler/mqtt_test.go +++ b/core/handler/mqtt_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/TheThingsNetwork/go-utils/log/apex" "github.com/TheThingsNetwork/ttn/core/component" "github.com/TheThingsNetwork/ttn/core/handler/device" "github.com/TheThingsNetwork/ttn/core/types" @@ -26,7 +25,7 @@ func TestHandleMQTT(t *testing.T) { a := New(t) var wg WaitGroup - c := mqtt.NewClient(apex.Wrap(GetLogger(t, "TestHandleMQTT")), "test", "", "", fmt.Sprintf("tcp://%s", host)) + c := mqtt.NewClient(GetLogger(t, "TestHandleMQTT"), "test", "", "", fmt.Sprintf("tcp://%s", host)) err := c.Connect() a.So(err, ShouldBeNil) appID := "handler-mqtt-app1" diff --git a/core/handler/types.go b/core/handler/types.go index 4ea614152..2095aa69d 100644 --- a/core/handler/types.go +++ b/core/handler/types.go @@ -8,15 +8,15 @@ import ( "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/utils/errors" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb_broker "github.com/TheThingsNetwork/ttn/api/broker" - "github.com/apex/log" ) // UplinkProcessor processes an uplink protobuf to an application-layer uplink message -type UplinkProcessor func(ctx log.Interface, ttnUp *pb_broker.DeduplicatedUplinkMessage, appUp *types.UplinkMessage, device *device.Device) error +type UplinkProcessor func(ctx ttnlog.Interface, ttnUp *pb_broker.DeduplicatedUplinkMessage, appUp *types.UplinkMessage, device *device.Device) error // DownlinkProcessor processes an application-layer downlink message to a downlik protobuf -type DownlinkProcessor func(ctx log.Interface, appDown *types.DownlinkMessage, ttnDown *pb_broker.DownlinkMessage, device *device.Device) error +type DownlinkProcessor func(ctx ttnlog.Interface, appDown *types.DownlinkMessage, ttnDown *pb_broker.DownlinkMessage, device *device.Device) error // ErrNotNeeded indicates that the processing of a message should be aborted var ErrNotNeeded = errors.New("Further processing not needed") diff --git a/core/handler/uplink.go b/core/handler/uplink.go index f49fc685f..e4ba21c4e 100644 --- a/core/handler/uplink.go +++ b/core/handler/uplink.go @@ -6,10 +6,10 @@ package handler import ( "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb_broker "github.com/TheThingsNetwork/ttn/api/broker" "github.com/TheThingsNetwork/ttn/api/trace" "github.com/TheThingsNetwork/ttn/core/types" - "github.com/apex/log" ) // ResponseDeadline indicates how long @@ -17,7 +17,7 @@ var ResponseDeadline = 100 * time.Millisecond func (h *handler) HandleUplink(uplink *pb_broker.DeduplicatedUplinkMessage) (err error) { appID, devID := uplink.AppId, uplink.DevId - ctx := h.Ctx.WithFields(log.Fields{ + ctx := h.Ctx.WithFields(ttnlog.Fields{ "AppID": appID, "DevID": devID, }) diff --git a/core/networkserver/device/device.go b/core/networkserver/device/device.go index 05dc22ecf..541f43204 100644 --- a/core/networkserver/device/device.go +++ b/core/networkserver/device/device.go @@ -20,18 +20,17 @@ type Options struct { // Device contains the state of a device type Device struct { - old *Device - DevEUI types.DevEUI `redis:"dev_eui"` - AppEUI types.AppEUI `redis:"app_eui"` - AppID string `redis:"app_id"` - DevID string `redis:"dev_id"` - DevAddr types.DevAddr `redis:"dev_addr"` - NwkSKey types.NwkSKey `redis:"nwk_s_key"` - FCntUp uint32 `redis:"f_cnt_up"` - FCntDown uint32 `redis:"f_cnt_down"` - LastSeen time.Time `redis:"last_seen"` - Options Options `redis:"options"` - Utilization Utilization `redis:"utilization"` + old *Device + DevEUI types.DevEUI `redis:"dev_eui"` + AppEUI types.AppEUI `redis:"app_eui"` + AppID string `redis:"app_id"` + DevID string `redis:"dev_id"` + DevAddr types.DevAddr `redis:"dev_addr"` + NwkSKey types.NwkSKey `redis:"nwk_s_key"` + FCntUp uint32 `redis:"f_cnt_up"` + FCntDown uint32 `redis:"f_cnt_down"` + LastSeen time.Time `redis:"last_seen"` + Options Options `redis:"options"` CreatedAt time.Time `redis:"created_at"` UpdatedAt time.Time `redis:"updated_at"` diff --git a/core/networkserver/device/store.go b/core/networkserver/device/store.go index 573110440..02cac0d0b 100644 --- a/core/networkserver/device/store.go +++ b/core/networkserver/device/store.go @@ -15,7 +15,7 @@ import ( // Store interface for Devices type Store interface { - List() ([]*Device, error) + List(opts *storage.ListOptions) ([]*Device, error) ListForAddress(devAddr types.DevAddr) ([]*Device, error) Get(appEUI types.AppEUI, devEUI types.DevEUI) (*Device, error) Set(new *Device, properties ...string) (err error) @@ -51,15 +51,15 @@ type RedisDeviceStore struct { } // List all Devices -func (s *RedisDeviceStore) List() ([]*Device, error) { - devicesI, err := s.store.List("", nil) +func (s *RedisDeviceStore) List(opts *storage.ListOptions) ([]*Device, error) { + devicesI, err := s.store.List("", opts) if err != nil { return nil, err } - devices := make([]*Device, 0, len(devicesI)) - for _, deviceI := range devicesI { + devices := make([]*Device, len(devicesI)) + for i, deviceI := range devicesI { if device, ok := deviceI.(Device); ok { - devices = append(devices, &device) + devices[i] = &device } } return devices, nil @@ -78,10 +78,10 @@ func (s *RedisDeviceStore) ListForAddress(devAddr types.DevAddr) ([]*Device, err if err != nil { return nil, err } - devices := make([]*Device, 0, len(devicesI)) - for _, deviceI := range devicesI { + devices := make([]*Device, len(devicesI)) + for i, deviceI := range devicesI { if device, ok := deviceI.(Device); ok { - devices = append(devices, &device) + devices[i] = &device } } return devices, nil diff --git a/core/networkserver/device/store_test.go b/core/networkserver/device/store_test.go index 616545cb1..1fa5f0a7f 100644 --- a/core/networkserver/device/store_test.go +++ b/core/networkserver/device/store_test.go @@ -108,7 +108,7 @@ func TestDeviceStore(t *testing.T) { a.So(dev.DevAddr, ShouldEqual, types.DevAddr{0, 0, 0, 3}) // List - devices, err := s.List() + devices, err := s.List(nil) a.So(err, ShouldBeNil) a.So(devices, ShouldHaveLength, 2) diff --git a/core/networkserver/device/utilization.go b/core/networkserver/device/utilization.go deleted file mode 100644 index 968479233..000000000 --- a/core/networkserver/device/utilization.go +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright © 2017 The Things Network -// Use of this source code is governed by the MIT license that can be found in the LICENSE file. - -package device - -// Utilization of the device -type Utilization interface{} diff --git a/core/networkserver/get_devices.go b/core/networkserver/get_devices.go index 1ea4e4cd2..10c57b425 100644 --- a/core/networkserver/get_devices.go +++ b/core/networkserver/get_devices.go @@ -22,6 +22,9 @@ func (n *networkServer) HandleGetDevices(req *pb.DevicesRequest) (*pb.DevicesRes } for _, device := range devices { + if device == nil { + continue + } fullFCnt := fcnt.GetFull(device.FCntUp, uint16(req.FCnt)) dev := &pb_lorawan.Device{ AppEui: &device.AppEUI, diff --git a/core/networkserver/manager_server.go b/core/networkserver/manager_server.go index 574093eb4..9eacdba43 100644 --- a/core/networkserver/manager_server.go +++ b/core/networkserver/manager_server.go @@ -7,6 +7,7 @@ import ( "fmt" "time" + "github.com/TheThingsNetwork/go-account-lib/claims" "github.com/TheThingsNetwork/go-account-lib/rights" pb "github.com/TheThingsNetwork/ttn/api/networkserver" pb_lorawan "github.com/TheThingsNetwork/ttn/api/protocol/lorawan" @@ -24,6 +25,13 @@ type networkServerManager struct { clientRate *ratelimit.Registry } +func checkAppRights(claims *claims.Claims, appID string, right rights.Right) error { + if !claims.AppRight(appID, right) { + return errors.NewErrPermissionDenied(fmt.Sprintf(`No "%s" rights to Application "%s"`, right, appID)) + } + return nil +} + func (n *networkServerManager) getDevice(ctx context.Context, in *pb_lorawan.DeviceIdentifier) (*device.Device, error) { if err := in.Validate(); err != nil { return nil, errors.Wrap(err, "Invalid Device Identifier") @@ -39,8 +47,9 @@ func (n *networkServerManager) getDevice(ctx context.Context, in *pb_lorawan.Dev if err != nil { return nil, err } - if !claims.AppRight(dev.AppID, rights.AppSettings) { - return nil, errors.NewErrPermissionDenied(fmt.Sprintf("No access to Application %s", dev.AppID)) + err = checkAppRights(claims, dev.AppID, rights.Devices) + if err != nil { + return nil, err } return dev, nil } @@ -72,11 +81,6 @@ func (n *networkServerManager) GetDevice(ctx context.Context, in *pb_lorawan.Dev } func (n *networkServerManager) SetDevice(ctx context.Context, in *pb_lorawan.Device) (*empty.Empty, error) { - dev, err := n.getDevice(ctx, &pb_lorawan.DeviceIdentifier{AppEui: in.AppEui, DevEui: in.DevEui}) - if err != nil && errors.GetErrType(err) != errors.NotFound { - return nil, err - } - if err := in.Validate(); err != nil { return nil, errors.Wrap(err, "Invalid Device") } @@ -85,8 +89,14 @@ func (n *networkServerManager) SetDevice(ctx context.Context, in *pb_lorawan.Dev if err != nil { return nil, err } - if !claims.AppRight(in.AppId, rights.AppSettings) { - return nil, errors.NewErrPermissionDenied(fmt.Sprintf("No access to Application %s", dev.AppID)) + err = checkAppRights(claims, in.AppId, rights.Devices) + if err != nil { + return nil, err + } + + dev, err := n.getDevice(ctx, &pb_lorawan.DeviceIdentifier{AppEui: in.AppEui, DevEui: in.DevEui}) + if err != nil && errors.GetErrType(err) != errors.NotFound { + return nil, err } if dev == nil { diff --git a/core/proxy/proxy.go b/core/proxy/proxy.go index f378865e0..88f271ff1 100644 --- a/core/proxy/proxy.go +++ b/core/proxy/proxy.go @@ -8,7 +8,7 @@ import ( "strconv" "strings" - "github.com/apex/log" + ttnlog "github.com/TheThingsNetwork/go-utils/log" ) type tokenProxier struct { @@ -33,12 +33,12 @@ func WithToken(handler http.Handler) http.Handler { } type logProxier struct { - ctx log.Interface + ctx ttnlog.Interface handler http.Handler } func (p *logProxier) ServeHTTP(res http.ResponseWriter, req *http.Request) { - p.ctx.WithFields(log.Fields{ + p.ctx.WithFields(ttnlog.Fields{ "RemoteAddress": req.RemoteAddr, "Method": req.Method, "URI": req.RequestURI, @@ -47,7 +47,7 @@ func (p *logProxier) ServeHTTP(res http.ResponseWriter, req *http.Request) { } // WithLogger wraps the handler so that each request gets logged -func WithLogger(handler http.Handler, ctx log.Interface) http.Handler { +func WithLogger(handler http.Handler, ctx ttnlog.Interface) http.Handler { return &logProxier{ctx, handler} } diff --git a/core/router/activation.go b/core/router/activation.go index ce11caa7f..ca50d5f68 100644 --- a/core/router/activation.go +++ b/core/router/activation.go @@ -8,6 +8,7 @@ import ( "sync" "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb_broker "github.com/TheThingsNetwork/ttn/api/broker" pb_protocol "github.com/TheThingsNetwork/ttn/api/protocol" pb_lorawan "github.com/TheThingsNetwork/ttn/api/protocol/lorawan" @@ -15,11 +16,10 @@ import ( "github.com/TheThingsNetwork/ttn/api/trace" "github.com/TheThingsNetwork/ttn/core/band" "github.com/TheThingsNetwork/ttn/utils/errors" - "github.com/apex/log" ) func (r *router) HandleActivation(gatewayID string, activation *pb.DeviceActivationRequest) (res *pb.DeviceActivationResponse, err error) { - ctx := r.Ctx.WithFields(log.Fields{ + ctx := r.Ctx.WithFields(ttnlog.Fields{ "GatewayID": gatewayID, "AppEUI": *activation.AppEui, "DevEUI": *activation.DevEui, diff --git a/core/router/downlink.go b/core/router/downlink.go index 6e664b7a2..909bef989 100644 --- a/core/router/downlink.go +++ b/core/router/downlink.go @@ -9,6 +9,7 @@ import ( "strings" "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb_broker "github.com/TheThingsNetwork/ttn/api/broker" pb_gateway "github.com/TheThingsNetwork/ttn/api/gateway" pb_protocol "github.com/TheThingsNetwork/ttn/api/protocol" @@ -20,11 +21,10 @@ import ( "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/utils/errors" "github.com/TheThingsNetwork/ttn/utils/toa" - "github.com/apex/log" ) func (r *router) SubscribeDownlink(gatewayID string, subscriptionID string) (<-chan *pb.DownlinkMessage, error) { - ctx := r.Ctx.WithFields(log.Fields{ + ctx := r.Ctx.WithFields(ttnlog.Fields{ "GatewayID": gatewayID, }) diff --git a/core/router/gateway/gateway.go b/core/router/gateway/gateway.go index 633d3623e..f5e3fbc23 100644 --- a/core/router/gateway/gateway.go +++ b/core/router/gateway/gateway.go @@ -7,14 +7,14 @@ import ( "sync" "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb "github.com/TheThingsNetwork/ttn/api/gateway" pb_monitor "github.com/TheThingsNetwork/ttn/api/monitor" pb_router "github.com/TheThingsNetwork/ttn/api/router" - "github.com/apex/log" ) // NewGateway creates a new in-memory Gateway structure -func NewGateway(ctx log.Interface, id string) *Gateway { +func NewGateway(ctx ttnlog.Interface, id string) *Gateway { ctx = ctx.WithField("GatewayID", id) gtw := &Gateway{ ID: id, @@ -42,7 +42,7 @@ type Gateway struct { Monitors pb_monitor.Registry - Ctx log.Interface + Ctx ttnlog.Interface } func (g *Gateway) SetAuth(token string, authenticated bool) { diff --git a/core/router/gateway/schedule.go b/core/router/gateway/schedule.go index e5493c266..a9f1c8dd2 100644 --- a/core/router/gateway/schedule.go +++ b/core/router/gateway/schedule.go @@ -9,13 +9,13 @@ import ( "sync/atomic" "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb_lorawan "github.com/TheThingsNetwork/ttn/api/protocol/lorawan" router_pb "github.com/TheThingsNetwork/ttn/api/router" "github.com/TheThingsNetwork/ttn/api/trace" "github.com/TheThingsNetwork/ttn/utils/errors" "github.com/TheThingsNetwork/ttn/utils/random" "github.com/TheThingsNetwork/ttn/utils/toa" - "github.com/apex/log" ) // Schedule is used to schedule downlink transmissions @@ -36,7 +36,7 @@ type Schedule interface { } // NewSchedule creates a new Schedule -func NewSchedule(ctx log.Interface) Schedule { +func NewSchedule(ctx ttnlog.Interface) Schedule { s := &schedule{ ctx: ctx, items: make(map[string]*scheduledItem), @@ -74,7 +74,7 @@ type scheduledItem struct { type schedule struct { sync.RWMutex - ctx log.Interface + ctx ttnlog.Interface offset int64 items map[string]*scheduledItem downlink chan *router_pb.DownlinkMessage diff --git a/core/router/server.go b/core/router/server.go index 0a501ca53..79ee3da9d 100644 --- a/core/router/server.go +++ b/core/router/server.go @@ -8,7 +8,6 @@ import ( "time" "github.com/TheThingsNetwork/go-account-lib/claims" - "github.com/TheThingsNetwork/go-utils/log/apex" "github.com/TheThingsNetwork/ttn/api" pb_gateway "github.com/TheThingsNetwork/ttn/api/gateway" "github.com/TheThingsNetwork/ttn/api/ratelimit" @@ -148,7 +147,7 @@ func (r *routerRPC) Activate(ctx context.Context, req *pb.DeviceActivationReques // RegisterRPC registers this router as a RouterServer (github.com/TheThingsNetwork/ttn/api/router) func (r *router) RegisterRPC(s *grpc.Server) { server := &routerRPC{router: r} - server.SetLogger(apex.Wrap(r.Ctx)) + server.SetLogger(r.Ctx) server.UplinkChanFunc = server.getUplink server.DownlinkChanFunc = server.getDownlink server.GatewayStatusChanFunc = server.getGatewayStatus diff --git a/core/router/uplink.go b/core/router/uplink.go index a95c6bccb..50195b9fb 100644 --- a/core/router/uplink.go +++ b/core/router/uplink.go @@ -6,13 +6,13 @@ package router import ( "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb_broker "github.com/TheThingsNetwork/ttn/api/broker" pb_lorawan "github.com/TheThingsNetwork/ttn/api/protocol/lorawan" pb "github.com/TheThingsNetwork/ttn/api/router" "github.com/TheThingsNetwork/ttn/api/trace" "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/utils/errors" - "github.com/apex/log" "github.com/brocaar/lorawan" ) @@ -43,7 +43,7 @@ func (r *router) HandleUplink(gatewayID string, uplink *pb.UplinkMessage) (err e } devEUI := types.DevEUI(joinRequestPayload.DevEUI) appEUI := types.AppEUI(joinRequestPayload.AppEUI) - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "DevEUI": devEUI, "AppEUI": appEUI, }).Debug("Handle Uplink as Activation") @@ -68,7 +68,7 @@ func (r *router) HandleUplink(gatewayID string, uplink *pb.UplinkMessage) (err e } if gateway := uplink.GatewayMetadata; gateway != nil { - ctx = ctx.WithFields(log.Fields{ + ctx = ctx.WithFields(ttnlog.Fields{ "Frequency": gateway.Frequency, "RSSI": gateway.Rssi, "SNR": gateway.Snr, @@ -81,7 +81,7 @@ func (r *router) HandleUplink(gatewayID string, uplink *pb.UplinkMessage) (err e } devAddr := types.DevAddr(macPayload.FHDR.DevAddr) - ctx = ctx.WithFields(log.Fields{ + ctx = ctx.WithFields(ttnlog.Fields{ "DevAddr": devAddr, "FCnt": macPayload.FHDR.FCnt, }) diff --git a/core/storage/conversions.go b/core/storage/conversions.go deleted file mode 100644 index b80509b8d..000000000 --- a/core/storage/conversions.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright © 2017 The Things Network -// Use of this source code is governed by the MIT license that can be found in the LICENSE file. - -package storage - -import ( - "encoding/hex" - "fmt" - "strconv" -) - -// FormatFloat32 does what its name suggests -func FormatFloat32(value float32) string { - return strconv.FormatFloat(float64(value), 'f', -1, 32) -} - -// FormatFloat64 does what its name suggests -func FormatFloat64(value float64) string { - return strconv.FormatFloat(value, 'f', -1, 64) -} - -// FormatInt32 does what its name suggests -func FormatInt32(value int32) string { - return FormatInt64(int64(value)) -} - -// FormatInt64 does what its name suggests -func FormatInt64(value int64) string { - return strconv.FormatInt(value, 10) -} - -// FormatUint32 does what its name suggests -func FormatUint32(value uint32) string { - return FormatUint64(uint64(value)) -} - -// FormatUint64 does what its name suggests -func FormatUint64(value uint64) string { - return strconv.FormatUint(value, 10) -} - -// FormatBool does what its name suggests -func FormatBool(value bool) string { - return strconv.FormatBool(value) -} - -// FormatBytes does what its name suggests -func FormatBytes(value []byte) string { - return fmt.Sprintf("%X", value) -} - -// ParseFloat32 does what its name suggests -func ParseFloat32(val string) (float32, error) { - res, err := strconv.ParseFloat(val, 32) - return float32(res), err -} - -// ParseFloat64 does what its name suggests -func ParseFloat64(val string) (float64, error) { - return strconv.ParseFloat(val, 64) -} - -// ParseInt32 does what its name suggests -func ParseInt32(val string) (int32, error) { - res, err := strconv.ParseInt(val, 10, 32) - return int32(res), err -} - -// ParseInt64 does what its name suggests -func ParseInt64(val string) (int64, error) { - return strconv.ParseInt(val, 10, 64) -} - -// ParseUint32 does what its name suggests -func ParseUint32(val string) (uint32, error) { - res, err := strconv.ParseUint(val, 10, 32) - return uint32(res), err -} - -// ParseUint64 does what its name suggests -func ParseUint64(val string) (uint64, error) { - return strconv.ParseUint(val, 10, 64) -} - -// ParseBool does what its name suggests -func ParseBool(val string) (bool, error) { - return strconv.ParseBool(val) -} - -// ParseBytes does what its name suggests -func ParseBytes(val string) ([]byte, error) { - return hex.DecodeString(val) -} diff --git a/core/storage/conversions_test.go b/core/storage/conversions_test.go deleted file mode 100644 index 7114216a0..000000000 --- a/core/storage/conversions_test.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright © 2017 The Things Network -// Use of this source code is governed by the MIT license that can be found in the LICENSE file. - -package storage - -import ( - "testing" - - "github.com/smartystreets/assertions" -) - -func TestFloat32(t *testing.T) { - a := assertions.New(t) - a.So(FormatFloat32(123.456), assertions.ShouldEqual, "123.456") - f32, err := ParseFloat32("123.456") - a.So(err, assertions.ShouldBeNil) - a.So(f32, assertions.ShouldEqual, 123.456) -} - -func TestFloat64(t *testing.T) { - a := assertions.New(t) - a.So(FormatFloat64(123.456), assertions.ShouldEqual, "123.456") - f64, err := ParseFloat64("123.456") - a.So(err, assertions.ShouldBeNil) - a.So(f64, assertions.ShouldEqual, 123.456) -} - -func TestInt32(t *testing.T) { - a := assertions.New(t) - a.So(FormatInt32(-123456), assertions.ShouldEqual, "-123456") - i32, err := ParseInt32("-123456") - a.So(err, assertions.ShouldBeNil) - a.So(i32, assertions.ShouldEqual, -123456) -} - -func TestInt64(t *testing.T) { - a := assertions.New(t) - a.So(FormatInt64(-123456), assertions.ShouldEqual, "-123456") - i64, err := ParseInt64("-123456") - a.So(err, assertions.ShouldBeNil) - a.So(i64, assertions.ShouldEqual, -123456) -} - -func TestUint32(t *testing.T) { - a := assertions.New(t) - a.So(FormatUint32(123456), assertions.ShouldEqual, "123456") - i32, err := ParseUint32("123456") - a.So(err, assertions.ShouldBeNil) - a.So(i32, assertions.ShouldEqual, 123456) -} - -func TestUint64(t *testing.T) { - a := assertions.New(t) - a.So(FormatUint64(123456), assertions.ShouldEqual, "123456") - i64, err := ParseUint64("123456") - a.So(err, assertions.ShouldBeNil) - a.So(i64, assertions.ShouldEqual, 123456) -} - -func TestBool(t *testing.T) { - a := assertions.New(t) - a.So(FormatBool(true), assertions.ShouldEqual, "true") - b, err := ParseBool("true") - a.So(err, assertions.ShouldBeNil) - a.So(b, assertions.ShouldEqual, true) - a.So(FormatBool(false), assertions.ShouldEqual, "false") - b, err = ParseBool("false") - a.So(err, assertions.ShouldBeNil) - a.So(b, assertions.ShouldEqual, false) -} - -func TestBytes(t *testing.T) { - a := assertions.New(t) - a.So(FormatBytes([]byte{0x12, 0x34, 0xcd, 0xef}), assertions.ShouldEqual, "1234CDEF") - i64, err := ParseBytes("1234CDEF") - a.So(err, assertions.ShouldBeNil) - a.So(i64, assertions.ShouldResemble, []byte{0x12, 0x34, 0xcd, 0xef}) -} diff --git a/core/storage/decoder.go b/core/storage/decoder.go deleted file mode 100644 index 0c48f6534..000000000 --- a/core/storage/decoder.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright © 2017 The Things Network -// Use of this source code is governed by the MIT license that can be found in the LICENSE file. - -package storage - -import ( - "encoding" - "encoding/json" - "fmt" - "reflect" - "strconv" - - "github.com/fatih/structs" -) - -// StringStringMapDecoder is used to decode a map[string]string to a struct -type StringStringMapDecoder func(input map[string]string) (interface{}, error) - -func decodeToType(typ reflect.Kind, value string) interface{} { - switch typ { - case reflect.String: - return value - case reflect.Bool: - v, _ := strconv.ParseBool(value) - return v - case reflect.Int: - v, _ := strconv.ParseInt(value, 10, 64) - return int(v) - case reflect.Int8: - return int8(decodeToType(reflect.Int, value).(int)) - case reflect.Int16: - return int16(decodeToType(reflect.Int, value).(int)) - case reflect.Int32: - return int32(decodeToType(reflect.Int, value).(int)) - case reflect.Int64: - return int64(decodeToType(reflect.Int, value).(int)) - case reflect.Uint: - v, _ := strconv.ParseUint(value, 10, 64) - return uint(v) - case reflect.Uint8: - return uint8(decodeToType(reflect.Uint, value).(uint)) - case reflect.Uint16: - return uint16(decodeToType(reflect.Uint, value).(uint)) - case reflect.Uint32: - return uint32(decodeToType(reflect.Uint, value).(uint)) - case reflect.Uint64: - return uint64(decodeToType(reflect.Uint, value).(uint)) - case reflect.Float64: - v, _ := strconv.ParseFloat(value, 64) - return v - case reflect.Float32: - return float32(decodeToType(reflect.Float64, value).(float64)) - } - return nil -} - -func unmarshalToType(typ reflect.Type, value string) (val interface{}, err error) { - // If we get a pointer in, we'll return a pointer out - if typ.Kind() == reflect.Ptr { - val = reflect.New(typ.Elem()).Interface() - } else { - val = reflect.New(typ).Interface() - } - defer func() { - if err == nil && typ.Kind() != reflect.Ptr { - val = reflect.Indirect(reflect.ValueOf(val)).Interface() - } - }() - - // If we can just assign the value, return the value - if typ.AssignableTo(reflect.TypeOf(value)) { - return value, nil - } - - // Try Unmarshalers - if um, ok := val.(encoding.TextUnmarshaler); ok { - if err = um.UnmarshalText([]byte(value)); err == nil { - return val, nil - } - } - if um, ok := val.(json.Unmarshaler); ok { - if err = um.UnmarshalJSON([]byte(value)); err == nil { - return val, nil - } - } - - // Try conversion - if typ.ConvertibleTo(reflect.TypeOf(value)) { - return reflect.ValueOf(value).Convert(typ).Interface(), nil - } - - // Try JSON - if err = json.Unmarshal([]byte(value), val); err == nil { - return val, nil - } - - // Return error if we have one - if err != nil { - return nil, err - } - - return val, fmt.Errorf("No way to unmarshal \"%s\" to %s", value, typ.Name()) -} - -// buildDefaultStructDecoder is used by the RedisMapStore -func buildDefaultStructDecoder(base interface{}, tagName string) StringStringMapDecoder { - if tagName == "" { - tagName = defaultTagName - } - - return func(input map[string]string) (output interface{}, err error) { - baseType := reflect.TypeOf(base) - // If we get a pointer in, we'll return a pointer out - if baseType.Kind() == reflect.Ptr { - output = reflect.New(baseType.Elem()).Interface() - } else { - output = reflect.New(baseType).Interface() - } - defer func() { - if err == nil && baseType.Kind() != reflect.Ptr { - output = reflect.Indirect(reflect.ValueOf(output)).Interface() - } - }() - - s := structs.New(output) - for _, field := range s.Fields() { - if !field.IsExported() { - continue - } - - tagName, _ := parseTag(field.Tag(tagName)) - if tagName == "" || tagName == "-" { - continue - } - if str, ok := input[tagName]; ok { - baseField, _ := baseType.FieldByName(field.Name()) - - if str == "" { - continue - } - - var val interface{} - switch field.Kind() { - case reflect.Ptr: - if str == "null" { - continue - } - fallthrough - case reflect.Struct, reflect.Array, reflect.Interface, reflect.Slice: - var err error - val, err = unmarshalToType(baseField.Type, str) - if err != nil { - return nil, err - } - default: - val = decodeToType(field.Kind(), str) - } - - if val == nil { - continue - } - - if !baseField.Type.AssignableTo(reflect.TypeOf(val)) && baseField.Type.ConvertibleTo(reflect.TypeOf(val)) { - val = reflect.ValueOf(val).Convert(baseField.Type).Interface() - } - - if err := field.Set(val); err != nil { - return nil, err - } - } - } - return output, nil - } -} diff --git a/core/storage/decoder_test.go b/core/storage/decoder_test.go deleted file mode 100644 index a9de0d6d5..000000000 --- a/core/storage/decoder_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright © 2017 The Things Network -// Use of this source code is governed by the MIT license that can be found in the LICENSE file. - -package storage - -import ( - "encoding/hex" - "reflect" - "strings" - "testing" - - "github.com/TheThingsNetwork/ttn/core/types" - . "github.com/smartystreets/assertions" -) - -func TestDecodeToType(t *testing.T) { - a := New(t) - a.So(decodeToType(reflect.String, "abc"), ShouldEqual, "abc") - a.So(decodeToType(reflect.Bool, "true"), ShouldEqual, true) - a.So(decodeToType(reflect.Bool, "false"), ShouldEqual, false) - a.So(decodeToType(reflect.Int, "-10"), ShouldEqual, -10) - a.So(decodeToType(reflect.Int8, "-10"), ShouldEqual, -10) - a.So(decodeToType(reflect.Int16, "-10"), ShouldEqual, -10) - a.So(decodeToType(reflect.Int32, "-10"), ShouldEqual, -10) - a.So(decodeToType(reflect.Int64, "-10"), ShouldEqual, -10) - a.So(decodeToType(reflect.Uint, "10"), ShouldEqual, 10) - a.So(decodeToType(reflect.Uint8, "10"), ShouldEqual, 10) - a.So(decodeToType(reflect.Uint16, "10"), ShouldEqual, 10) - a.So(decodeToType(reflect.Uint32, "10"), ShouldEqual, 10) - a.So(decodeToType(reflect.Uint64, "10"), ShouldEqual, 10) - a.So(decodeToType(reflect.Float64, "12.34"), ShouldEqual, 12.34) - a.So(decodeToType(reflect.Float32, "12.34"), ShouldEqual, 12.34) - a.So(decodeToType(reflect.Struct, "blabla"), ShouldBeNil) -} - -type noIdea struct { - something complex128 -} - -type strtp string - -type testTextUnmarshaler struct { - Data string -} - -func (um *testTextUnmarshaler) UnmarshalText(text []byte) error { - um.Data = string(text) - return nil -} - -type testCustomType [2]byte - -type testJSONUnmarshaler struct { - Customs []testCustomType -} - -type jsonStruct struct { - String string `json:"string"` -} - -func (jum *testJSONUnmarshaler) UnmarshalJSON(text []byte) error { - jum.Customs = []testCustomType{} - txt := strings.Trim(string(text), "[]") - txtlist := strings.Split(txt, ",") - for _, txtitem := range txtlist { - txtitem = strings.Trim(txtitem, `"`) - b, _ := hex.DecodeString(txtitem) - var o testCustomType - copy(o[:], b[:]) - jum.Customs = append(jum.Customs, o) - } - return nil -} - -func TestUnmarshalToType(t *testing.T) { - a := New(t) - - var str string - strOut, err := unmarshalToType(reflect.TypeOf(str), "data") - a.So(err, ShouldBeNil) - a.So(strOut, ShouldEqual, "data") - - var strtp strtp - strtpOut, err := unmarshalToType(reflect.TypeOf(strtp), "data") - a.So(err, ShouldBeNil) - a.So(strtpOut, ShouldEqual, "data") - - var um testTextUnmarshaler - umOut, err := unmarshalToType(reflect.TypeOf(um), "data") - a.So(err, ShouldBeNil) - a.So(umOut.(testTextUnmarshaler), ShouldResemble, testTextUnmarshaler{"data"}) - - var jum testJSONUnmarshaler - jumOut, err := unmarshalToType(reflect.TypeOf(jum), `["abcd","1234","def0"]`) - a.So(err, ShouldBeNil) - a.So(jumOut.(testJSONUnmarshaler), ShouldResemble, testJSONUnmarshaler{[]testCustomType{ - testCustomType{0xab, 0xcd}, - testCustomType{0x12, 0x34}, - testCustomType{0xde, 0xf0}, - }}) - - var js jsonStruct - jsOut, err := unmarshalToType(reflect.TypeOf(js), `{"string": "String"}`) - a.So(err, ShouldBeNil) - a.So(jsOut.(jsonStruct), ShouldResemble, jsonStruct{"String"}) - - _, err = unmarshalToType(reflect.TypeOf(js), `this is no json`) - a.So(err, ShouldNotBeNil) - - var eui types.DevEUI - euiOut, err := unmarshalToType(reflect.TypeOf(eui), "0102abcd0304abcd") - a.So(err, ShouldBeNil) - a.So(euiOut.(types.DevEUI), ShouldEqual, types.DevEUI{0x01, 0x02, 0xab, 0xcd, 0x03, 0x04, 0xab, 0xcd}) - - var euiPtr *types.DevEUI - euiPtrOut, err := unmarshalToType(reflect.TypeOf(euiPtr), "0102abcd0304abcd") - a.So(err, ShouldBeNil) - a.So(euiPtrOut.(*types.DevEUI), ShouldResemble, &types.DevEUI{0x01, 0x02, 0xab, 0xcd, 0x03, 0x04, 0xab, 0xcd}) -} diff --git a/core/storage/encoder.go b/core/storage/encoder.go deleted file mode 100644 index 15c626b65..000000000 --- a/core/storage/encoder.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright © 2017 The Things Network -// Use of this source code is governed by the MIT license that can be found in the LICENSE file. - -package storage - -import ( - "encoding" - "encoding/json" - "fmt" - "reflect" - - "github.com/fatih/structs" -) - -// StringStringMapEncoder encodes the given properties of the input to a map[string]string for storage in Redis -type StringStringMapEncoder func(input interface{}, properties ...string) (map[string]string, error) - -type isZeroer interface { - IsZero() bool -} - -type isEmptier interface { - IsEmpty() bool -} - -func buildDefaultStructEncoder(tagName string) StringStringMapEncoder { - if tagName == "" { - tagName = defaultTagName - } - - return func(input interface{}, properties ...string) (map[string]string, error) { - vmap := make(map[string]string) - s := structs.New(input) - s.TagName = tagName - if len(properties) == 0 { - properties = s.Names() - } - for _, field := range s.Fields() { - if !field.IsExported() { - continue - } - - if !stringInSlice(field.Name(), properties) { - continue - } - - tagName, opts := parseTag(field.Tag(tagName)) - if tagName == "" || tagName == "-" { - continue - } - - val := field.Value() - - if opts.Has("omitempty") { - if field.IsZero() { - continue - } - if z, ok := val.(isZeroer); ok && z.IsZero() { - continue - } - if z, ok := val.(isEmptier); ok && z.IsEmpty() { - continue - } - } - - if v, ok := val.(string); ok { - vmap[tagName] = v - continue - } - - if !field.IsZero() { - if m, ok := val.(encoding.TextMarshaler); ok { - txt, err := m.MarshalText() - if err != nil { - return nil, err - } - vmap[tagName] = string(txt) - continue - } - if m, ok := val.(json.Marshaler); ok { - txt, err := m.MarshalJSON() - if err != nil { - return nil, err - } - vmap[tagName] = string(txt) - continue - } - } - - if field.Kind() == reflect.String { - vmap[tagName] = fmt.Sprint(val) - continue - } - - if txt, err := json.Marshal(val); err == nil { - vmap[tagName] = string(txt) - continue - } - - vmap[tagName] = fmt.Sprintf("%v", val) - } - return vmap, nil - } -} diff --git a/core/storage/encoder_test.go b/core/storage/encoder_test.go deleted file mode 100644 index 656c92122..000000000 --- a/core/storage/encoder_test.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright © 2017 The Things Network -// Use of this source code is governed by the MIT license that can be found in the LICENSE file. - -package storage - -import ( - "testing" - "time" - - "github.com/TheThingsNetwork/ttn/core/types" - . "github.com/smartystreets/assertions" -) - -type strtps []strtp - -type jm struct { - txt string -} - -func (j jm) MarshalJSON() ([]byte, error) { - return []byte("{" + j.txt + "}"), nil -} - -type testStruct struct { - unexported string `redis:"unexported"` - NoRedis string `` - DisRedis string `redis:"-"` - String string `redis:"string"` - Strings []string `redis:"strings"` - EmptyString string `redis:"empty_string,omitempty"` - EmptyStrings []string `redis:"empty_strings,omitempty"` - AppEUI types.AppEUI `redis:"app_eui"` - AppEUIPtr *types.AppEUI `redis:"app_eui_ptr"` - EmptyAppEUI types.AppEUI `redis:"empty_app_eui,omitempty"` - EmptyAppEUIPtr *types.AppEUI `redis:"empty_app_eui_ptr,omitempty"` - Time time.Time `redis:"time"` - TimePtr *time.Time `redis:"time_ptr"` - EmptyTime time.Time `redis:"empty_time,omitempty"` - EmptyTimePtr *time.Time `redis:"empty_time_ptr,omitempty"` - STime Time `redis:"stime"` - STimePtr *Time `redis:"stime_ptr"` - EmptySTime Time `redis:"empty_stime,omitempty"` - EmptySTimePtr *Time `redis:"empty_stime_ptr,omitempty"` - Str strtp `redis:"str"` - Strs []strtp `redis:"strs"` - JM jm `redis:"jm"` - JMs []jm `redis:"jms"` - Int int `redis:"int"` - Uint uint `redis:"uint"` -} - -func TestDefaultStructEncoder(t *testing.T) { - a := New(t) - - var emptyAppEUI types.AppEUI - var appEUI = types.AppEUI{1, 2, 3, 4, 5, 6, 7, 8} - var now = time.Now() - var emptyTime time.Time - var stime = Time{now} - var emptySTime Time - - out, err := buildDefaultStructEncoder("")(&testStruct{ - unexported: "noop", - NoRedis: "noop", - DisRedis: "noop", - String: "string", - Strings: []string{"string1", "string2"}, - AppEUI: appEUI, - AppEUIPtr: &appEUI, - EmptyAppEUIPtr: &emptyAppEUI, - Time: now, - TimePtr: &now, - STime: stime, - STimePtr: &stime, - EmptySTimePtr: &emptySTime, - EmptyTimePtr: &emptyTime, - JM: jm{"cool"}, - }) - a.So(err, ShouldBeNil) - a.So(out, ShouldNotContainKey, "unexported") - a.So(out, ShouldNotContainKey, "empty_string") - a.So(out, ShouldNotContainKey, "empty_strings") - a.So(out, ShouldNotContainKey, "empty_app_eui") - a.So(out, ShouldNotContainKey, "empty_app_eui_ptr") - a.So(out, ShouldNotContainKey, "empty_time") - a.So(out, ShouldNotContainKey, "empty_time_ptr") - a.So(out, ShouldNotContainKey, "empty_stime") - a.So(out, ShouldNotContainKey, "empty_stime_ptr") - a.So(out["string"], ShouldEqual, "string") - a.So(out["strings"], ShouldEqual, `["string1","string2"]`) - a.So(out["jm"], ShouldEqual, "{cool}") - - out, err = buildDefaultStructEncoder("")(&testStruct{ - String: "noop", - Strings: []string{"string1", "string2"}, - }, "String") - a.So(err, ShouldBeNil) - a.So(out, ShouldNotContainKey, "strings") -} diff --git a/core/storage/redis_kv_store.go b/core/storage/redis_kv_store.go index bfa6d6f2b..b2bb251be 100644 --- a/core/storage/redis_kv_store.go +++ b/core/storage/redis_kv_store.go @@ -44,6 +44,9 @@ func (s *RedisKVStore) GetAll(keys []string, options *ListOptions) (map[string]s sort.Strings(keys) selectedKeys := selectKeys(keys, options) + if len(selectedKeys) == 0 { + return map[string]string{}, nil + } pipe := s.client.Pipeline() defer pipe.Close() @@ -80,11 +83,20 @@ func (s *RedisKVStore) List(selector string, options *ListOptions) (map[string]s if !strings.HasPrefix(selector, s.prefix) { selector = s.prefix + selector } - keys, err := s.client.Keys(selector).Result() - if err != nil { - return nil, err + var allKeys []string + var cursor uint64 + for { + keys, next, err := s.client.Scan(cursor, selector, 0).Result() + if err != nil { + return nil, err + } + allKeys = append(allKeys, keys...) + cursor = next + if cursor == 0 { + break + } } - return s.GetAll(keys, options) + return s.GetAll(allKeys, options) } // Get one result, prepending the prefix to the key if necessary diff --git a/core/storage/redis_map_store.go b/core/storage/redis_map_store.go index 6310b1b7a..2492ca083 100644 --- a/core/storage/redis_map_store.go +++ b/core/storage/redis_map_store.go @@ -7,8 +7,8 @@ import ( "sort" "strings" + "github.com/TheThingsNetwork/go-utils/encoding" "github.com/TheThingsNetwork/ttn/utils/errors" - redis "gopkg.in/redis.v5" ) @@ -16,8 +16,8 @@ import ( type RedisMapStore struct { prefix string client *redis.Client - encoder StringStringMapEncoder - decoder StringStringMapDecoder + encoder func(input interface{}, properties ...string) (map[string]string, error) + decoder func(input map[string]string) (output interface{}, err error) } // NewRedisMapStore returns a new RedisMapStore that talks to the given Redis client and respects the given prefix @@ -33,17 +33,24 @@ func NewRedisMapStore(client *redis.Client, prefix string) *RedisMapStore { // SetBase sets the base struct for automatically encoding and decoding to and from Redis format func (s *RedisMapStore) SetBase(base interface{}, tagName string) { - s.SetEncoder(buildDefaultStructEncoder(tagName)) - s.SetDecoder(buildDefaultStructDecoder(base, tagName)) + if tagName == "" { + tagName = "redis" + } + s.SetEncoder(func(input interface{}, properties ...string) (map[string]string, error) { + return encoding.ToStringStringMap(tagName, input, properties...) + }) + s.SetDecoder(func(input map[string]string) (output interface{}, err error) { + return encoding.FromStringStringMap(tagName, base, input) + }) } // SetEncoder sets the encoder to convert structs to Redis format -func (s *RedisMapStore) SetEncoder(encoder StringStringMapEncoder) { +func (s *RedisMapStore) SetEncoder(encoder func(input interface{}, properties ...string) (map[string]string, error)) { s.encoder = encoder } // SetDecoder sets the decoder to convert structs from Redis format -func (s *RedisMapStore) SetDecoder(decoder StringStringMapDecoder) { +func (s *RedisMapStore) SetDecoder(decoder func(input map[string]string) (output interface{}, err error)) { s.decoder = decoder } @@ -62,6 +69,9 @@ func (s *RedisMapStore) GetAll(keys []string, options *ListOptions) ([]interface sort.Strings(keys) selectedKeys := selectKeys(keys, options) + if len(selectedKeys) == 0 { + return []interface{}{}, nil + } pipe := s.client.Pipeline() defer pipe.Close() @@ -79,11 +89,11 @@ func (s *RedisMapStore) GetAll(keys []string, options *ListOptions) ([]interface } // Get all results from pipeline - results := make([]interface{}, 0, len(selectedKeys)) - for _, key := range selectedKeys { + results := make([]interface{}, len(selectedKeys)) + for i, key := range selectedKeys { if result, err := cmds[key].Result(); err == nil { if result, err := s.decoder(result); err == nil { - results = append(results, result) + results[i] = result } } } @@ -99,11 +109,20 @@ func (s *RedisMapStore) List(selector string, options *ListOptions) ([]interface if !strings.HasPrefix(selector, s.prefix) { selector = s.prefix + selector } - keys, err := s.client.Keys(selector).Result() - if err != nil { - return nil, err + var allKeys []string + var cursor uint64 + for { + keys, next, err := s.client.Scan(cursor, selector, 0).Result() + if err != nil { + return nil, err + } + allKeys = append(allKeys, keys...) + cursor = next + if cursor == 0 { + break + } } - return s.GetAll(keys, options) + return s.GetAll(allKeys, options) } // Get one result, prepending the prefix to the key if necessary diff --git a/core/storage/redis_set_store.go b/core/storage/redis_set_store.go index 52702641b..cdd90322a 100644 --- a/core/storage/redis_set_store.go +++ b/core/storage/redis_set_store.go @@ -44,6 +44,9 @@ func (s *RedisSetStore) GetAll(keys []string, options *ListOptions) (map[string] sort.Strings(keys) selectedKeys := selectKeys(keys, options) + if len(selectedKeys) == 0 { + return map[string][]string{}, nil + } pipe := s.client.Pipeline() defer pipe.Close() @@ -81,11 +84,20 @@ func (s *RedisSetStore) List(selector string, options *ListOptions) (map[string] if !strings.HasPrefix(selector, s.prefix) { selector = s.prefix + selector } - keys, err := s.client.Keys(selector).Result() - if err != nil { - return nil, err + var allKeys []string + var cursor uint64 + for { + keys, next, err := s.client.Scan(cursor, selector, 0).Result() + if err != nil { + return nil, err + } + allKeys = append(allKeys, keys...) + cursor = next + if cursor == 0 { + break + } } - return s.GetAll(keys, options) + return s.GetAll(allKeys, options) } // Get one result, prepending the prefix to the key if necessary @@ -118,9 +130,9 @@ func (s *RedisSetStore) Add(key string, values ...string) error { if !strings.HasPrefix(key, s.prefix) { key = s.prefix + key } - var valuesI []interface{} - for _, v := range values { - valuesI = append(valuesI, v) + valuesI := make([]interface{}, len(values)) + for i, v := range values { + valuesI[i] = v } return s.client.SAdd(key, valuesI...).Err() } @@ -130,9 +142,9 @@ func (s *RedisSetStore) Remove(key string, values ...string) error { if !strings.HasPrefix(key, s.prefix) { key = s.prefix + key } - var valuesI []interface{} - for _, v := range values { - valuesI = append(valuesI, v) + valuesI := make([]interface{}, len(values)) + for i, v := range values { + valuesI[i] = v } return s.client.SRem(key, valuesI...).Err() } diff --git a/core/storage/tags.go b/core/storage/tags.go deleted file mode 100644 index 379d31ecd..000000000 --- a/core/storage/tags.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright © 2017 The Things Network -// Use of this source code is governed by the MIT license that can be found in the LICENSE file. - -package storage - -import "strings" - -var defaultTagName = "redis" - -type tagOptions []string - -// Has returns true if opt is one of the options -func (t tagOptions) Has(opt string) bool { - for _, opt := range t { - if opt == opt { - return true - } - } - return false -} - -func parseTag(tag string) (string, tagOptions) { - res := strings.Split(tag, ",") - return res[0], res[1:] -} diff --git a/core/storage/util.go b/core/storage/util.go index db6209534..2ce6785f1 100644 --- a/core/storage/util.go +++ b/core/storage/util.go @@ -7,22 +7,32 @@ package storage type ListOptions struct { Limit int Offset int + + total int + selected int +} + +// GetTotalAndSelected returns the total number of items, along with the number of selected items +func (o ListOptions) GetTotalAndSelected() (total, selected int) { + return o.total, o.selected } func selectKeys(keys []string, options *ListOptions) []string { var start int var end = len(keys) if options != nil { - if options.Offset >= len(keys) { + options.total = end + if options.Offset >= options.total { return []string{} } start = options.Offset if options.Limit > 0 { - if options.Offset+options.Limit > len(keys) { - options.Limit = len(keys) - options.Offset + if options.Offset+options.Limit > options.total { + options.Limit = options.total - options.Offset } end = options.Offset + options.Limit } + options.selected = end - start } return keys[start:end] } diff --git a/mqtt/utils_test.go b/mqtt/utils_test.go index a155a9594..42aedf447 100644 --- a/mqtt/utils_test.go +++ b/mqtt/utils_test.go @@ -7,10 +7,9 @@ import ( "testing" "github.com/TheThingsNetwork/go-utils/log" - "github.com/TheThingsNetwork/go-utils/log/apex" tt "github.com/TheThingsNetwork/ttn/utils/testing" ) func getLogger(t *testing.T, tag string) log.Interface { - return apex.Wrap(tt.GetLogger(t, tag)) + return tt.GetLogger(t, tag) } diff --git a/ttnctl/cmd/applications_pf_set.go b/ttnctl/cmd/applications_pf_set.go index a2ea50055..01dc308d5 100644 --- a/ttnctl/cmd/applications_pf_set.go +++ b/ttnctl/cmd/applications_pf_set.go @@ -9,9 +9,9 @@ import ( "os" "strings" + "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/api/handler" "github.com/TheThingsNetwork/ttn/ttnctl/util" - "github.com/apex/log" "github.com/spf13/cobra" ) @@ -44,6 +44,15 @@ function Decoder(bytes, port) { return decoded; } + +Do you want to test the payload functions? (Y/n) +Y + +Payload: 12 34 +Port: 1 + + INFO Function tested successfully + INFO Updated application AppID=test `, Run: func(cmd *cobra.Command, args []string) { @@ -72,6 +81,12 @@ function Decoder(bytes, port) { if err != nil { ctx.WithError(err).Fatal("Could not read function file") } + fmt.Println(fmt.Sprintf(` +Function read from %s: + +%s +`, args[1], string(content))) + switch function { case "decoder": app.Decoder = string(content) @@ -99,7 +114,7 @@ function Decoder(bytes, port) { return decoded; } ########## Write your Decoder here and end with Ctrl+D (EOF):`) - app.Decoder = readFunction() + app.Decoder = readFunction(ctx) case "converter": fmt.Println(`function Converter(decoded, port) { // Merge, split or otherwise @@ -113,7 +128,7 @@ function Decoder(bytes, port) { return converted; } ########## Write your Converter here and end with Ctrl+D (EOF):`) - app.Converter = readFunction() + app.Converter = readFunction(ctx) case "validator": fmt.Println(`function Validator(converted, port) { // Return false if the decoded, converted @@ -126,7 +141,7 @@ function Decoder(bytes, port) { return true; } ########## Write your Validator here and end with Ctrl+D (EOF):`) - app.Validator = readFunction() + app.Validator = readFunction(ctx) case "encoder": fmt.Println(`function Encoder(object, port) { // Encode downlink messages sent as @@ -140,7 +155,54 @@ function Decoder(bytes, port) { return bytes; } ########## Write your Encoder here and end with Ctrl+D (EOF):`) - app.Encoder = readFunction() + app.Encoder = readFunction(ctx) + default: + ctx.Fatalf("Function %s does not exist", function) + } + } + + fmt.Printf("\nDo you want to test the payload functions? (Y/n)\n") + var response string + fmt.Scanln(&response) + + if strings.ToLower(response) == "y" || strings.ToLower(response) == "yes" || response == "" { + switch function { + case "decoder", "converter", "validator": + payload, err := util.ReadPayload() + if err != nil { + ctx.WithError(err).Fatal("Could not parse the payload") + } + + port, err := util.ReadPort() + if err != nil { + ctx.WithError(err).Fatal("Could not parse the port") + } + + result, err := manager.DryUplink(payload, app, uint32(port)) + if err != nil { + ctx.WithError(err).Fatal("Could not set the payload function") + } + + if !result.Valid { + ctx.Fatal("Could not set the payload function: Invalid result") + } + ctx.Infof("Function tested successfully. Object returned by the converter: %s", result.Fields) + case "encoder": + fields, err := util.ReadFields() + if err != nil { + ctx.WithError(err).Fatal("Could not parse the fields") + } + + port, err := util.ReadPort() + if err != nil { + ctx.WithError(err).Fatal("Could not parse the port") + } + + result, err := manager.DryDownlinkWithFields(fields, app, uint32(port)) + if err != nil { + ctx.WithError(err).Fatal("Could not set the payload function") + } + ctx.Infof("Function tested successfully. Encoded message: %v", result.Payload) default: ctx.Fatalf("Function %s does not exist", function) } @@ -157,14 +219,14 @@ function Decoder(bytes, port) { }, } -func readFunction() string { +func init() { + applicationsPayloadFunctionsCmd.AddCommand(applicationsPayloadFunctionsSetCmd) +} + +func readFunction(ctx log.Interface) string { content, err := ioutil.ReadAll(os.Stdin) if err != nil { ctx.WithError(err).Fatal("Could not read function from STDIN.") } return strings.TrimSpace(string(content)) } - -func init() { - applicationsPayloadFunctionsCmd.AddCommand(applicationsPayloadFunctionsSetCmd) -} diff --git a/ttnctl/cmd/applications_register.go b/ttnctl/cmd/applications_register.go index 8e980200a..bb5c9d2b0 100644 --- a/ttnctl/cmd/applications_register.go +++ b/ttnctl/cmd/applications_register.go @@ -4,8 +4,8 @@ package cmd import ( + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/ttnctl/util" - "github.com/apex/log" "github.com/spf13/cobra" ) @@ -30,7 +30,7 @@ var applicationsRegisterCmd = &cobra.Command{ ctx.WithError(err).Fatal("Could not register application") } - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "AppID": appID, }).Infof("Registered application") }, diff --git a/ttnctl/cmd/applications_select.go b/ttnctl/cmd/applications_select.go index 9efa20ab6..bde72ff98 100644 --- a/ttnctl/cmd/applications_select.go +++ b/ttnctl/cmd/applications_select.go @@ -6,9 +6,9 @@ package cmd import ( "fmt" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/ttnctl/util" - "github.com/apex/log" "github.com/gosuri/uitable" "github.com/spf13/cobra" ) @@ -116,7 +116,7 @@ var applicationsSelectCmd = &cobra.Command{ util.SetApp(ctx, app.ID, eui) - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "AppID": app.ID, "AppEUI": eui, }).Info("Updated configuration") diff --git a/ttnctl/cmd/applications_unregister.go b/ttnctl/cmd/applications_unregister.go index c1ac58558..cc84835f2 100644 --- a/ttnctl/cmd/applications_unregister.go +++ b/ttnctl/cmd/applications_unregister.go @@ -6,8 +6,8 @@ package cmd import ( "fmt" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/ttnctl/util" - "github.com/apex/log" "github.com/spf13/cobra" ) @@ -39,7 +39,7 @@ Are you sure you want to unregister application test? ctx.WithError(err).Fatal("Could not unregister application") } - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "AppID": appID, }).Infof("Unregistered application") }, diff --git a/ttnctl/cmd/devices.go b/ttnctl/cmd/devices.go index 029e95f40..cb34ceac6 100644 --- a/ttnctl/cmd/devices.go +++ b/ttnctl/cmd/devices.go @@ -4,8 +4,8 @@ package cmd import ( + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/ttnctl/util" - "github.com/apex/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -18,7 +18,7 @@ var devicesCmd = &cobra.Command{ PersistentPreRun: func(cmd *cobra.Command, args []string) { RootCmd.PersistentPreRun(cmd, args) util.GetAccount(ctx) - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "AppID": util.GetAppID(ctx), "AppEUI": util.GetAppEUI(ctx), }).Info("Using Application") diff --git a/ttnctl/cmd/devices_delete.go b/ttnctl/cmd/devices_delete.go index 653f9245a..8caa97c06 100644 --- a/ttnctl/cmd/devices_delete.go +++ b/ttnctl/cmd/devices_delete.go @@ -6,9 +6,9 @@ package cmd import ( "fmt" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/api" "github.com/TheThingsNetwork/ttn/ttnctl/util" - "github.com/apex/log" "github.com/spf13/cobra" ) @@ -51,7 +51,7 @@ Are you sure you want to delete device test from application test? ctx.WithError(err).Fatal("Could not delete device.") } - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "AppID": appID, "DevID": devID, }).Info("Deleted device") diff --git a/ttnctl/cmd/devices_list.go b/ttnctl/cmd/devices_list.go index 794908a12..8c0c392cd 100644 --- a/ttnctl/cmd/devices_list.go +++ b/ttnctl/cmd/devices_list.go @@ -6,8 +6,8 @@ package cmd import ( "fmt" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/ttnctl/util" - "github.com/apex/log" "github.com/gosuri/uitable" "github.com/spf13/cobra" ) @@ -34,7 +34,7 @@ test 70B3D57EF0000024 0001D544B2936FCE 26001ADA conn, manager := util.GetHandlerManager(ctx, appID) defer conn.Close() - devices, err := manager.GetDevicesForApplication(appID) + devices, err := manager.GetDevicesForApplication(appID, 0, 0) if err != nil { ctx.WithError(err).Fatal("Could not get devices.") } @@ -58,7 +58,7 @@ test 70B3D57EF0000024 0001D544B2936FCE 26001ADA fmt.Println(table) fmt.Println() - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "AppID": appID, }).Infof("Listed %d devices", len(devices)) }, diff --git a/ttnctl/cmd/devices_personalize.go b/ttnctl/cmd/devices_personalize.go index aae9fb24c..2f9fae5ed 100644 --- a/ttnctl/cmd/devices_personalize.go +++ b/ttnctl/cmd/devices_personalize.go @@ -6,11 +6,11 @@ package cmd import ( "strings" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/api" "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/ttnctl/util" "github.com/TheThingsNetwork/ttn/utils/random" - "github.com/apex/log" "github.com/spf13/cobra" ) @@ -99,7 +99,7 @@ var devicesPersonalizeCmd = &cobra.Command{ ctx.WithError(err).Fatal("Could not update Device") } - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "AppID": appID, "DevID": devID, "DevAddr": devAddr, diff --git a/ttnctl/cmd/devices_register.go b/ttnctl/cmd/devices_register.go index b451fcbdc..2b57d6099 100644 --- a/ttnctl/cmd/devices_register.go +++ b/ttnctl/cmd/devices_register.go @@ -4,13 +4,13 @@ package cmd import ( + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/api" "github.com/TheThingsNetwork/ttn/api/handler" "github.com/TheThingsNetwork/ttn/api/protocol/lorawan" "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/ttnctl/util" "github.com/TheThingsNetwork/ttn/utils/random" - "github.com/apex/log" "github.com/spf13/cobra" ) @@ -94,7 +94,7 @@ var devicesRegisterCmd = &cobra.Command{ ctx.WithError(err).Fatal("Could not register Device") } - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "AppID": appID, "DevID": devID, "AppEUI": appEUI, diff --git a/ttnctl/cmd/devices_set.go b/ttnctl/cmd/devices_set.go index 5cf6f1ea1..88b41f3f5 100644 --- a/ttnctl/cmd/devices_set.go +++ b/ttnctl/cmd/devices_set.go @@ -6,10 +6,10 @@ package cmd import ( "os" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/api" "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/ttnctl/util" - "github.com/apex/log" "github.com/spf13/cobra" ) @@ -157,7 +157,7 @@ var devicesSetCmd = &cobra.Command{ ctx.WithError(err).Fatal("Could not update Device") } - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "AppID": appID, "DevID": devID, }).Info("Updated device") diff --git a/ttnctl/cmd/join.go b/ttnctl/cmd/join.go index 3ade1744b..aae29d8d8 100644 --- a/ttnctl/cmd/join.go +++ b/ttnctl/cmd/join.go @@ -6,12 +6,12 @@ package cmd import ( "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" pb_lorawan "github.com/TheThingsNetwork/ttn/api/protocol/lorawan" "github.com/TheThingsNetwork/ttn/api/router" "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/ttnctl/util" "github.com/TheThingsNetwork/ttn/utils/otaa" - "github.com/apex/log" "github.com/brocaar/lorawan" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -118,7 +118,7 @@ var joinCmd = &cobra.Command{ appSKey, nwkSKey, _ := otaa.CalculateSessionKeys(appKey, accept.AppNonce, accept.NetId, devNonce) - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "DevAddr": accept.DevAddr, "NwkSKey": nwkSKey, "AppSKey": appSKey, diff --git a/ttnctl/cmd/root.go b/ttnctl/cmd/root.go index 72e99ddb9..b22ae7d6b 100644 --- a/ttnctl/cmd/root.go +++ b/ttnctl/cmd/root.go @@ -23,7 +23,7 @@ var cfgFile string var dataDir string var debug bool -var ctx log.Interface +var ctx ttnlog.Interface // RootCmd is the entrypoint for ttnctl var RootCmd = &cobra.Command{ @@ -35,18 +35,18 @@ var RootCmd = &cobra.Command{ if viper.GetBool("debug") { logLevel = log.DebugLevel } - ctx = &log.Logger{ + + ctx = apex.Wrap(&log.Logger{ Level: logLevel, Handler: cliHandler.New(os.Stdout), - } + }) if viper.GetBool("debug") { util.PrintConfig(ctx, true) } - ttnlog.Set(apex.Wrap(ctx)) + ttnlog.Set(ctx) grpclog.SetLogger(grpc.Wrap(ttnlog.Get())) - }, } diff --git a/ttnctl/cmd/uplink.go b/ttnctl/cmd/uplink.go index bb740eaef..674c8b45e 100644 --- a/ttnctl/cmd/uplink.go +++ b/ttnctl/cmd/uplink.go @@ -7,10 +7,10 @@ import ( "strconv" "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/api/router" "github.com/TheThingsNetwork/ttn/core/types" "github.com/TheThingsNetwork/ttn/ttnctl/util" - "github.com/apex/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -117,7 +117,7 @@ var uplinkCmd = &cobra.Command{ if err := m.Unmarshal(downlinkMessage.Payload); err != nil { ctx.WithError(err).Fatal("Could not unmarshal downlink") } - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "Payload": m.Payload, "FCnt": m.FCnt, "FPort": m.FPort, diff --git a/ttnctl/cmd/version.go b/ttnctl/cmd/version.go index 8477dc1a3..363f6d0df 100644 --- a/ttnctl/cmd/version.go +++ b/ttnctl/cmd/version.go @@ -8,8 +8,8 @@ package cmd import ( "time" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/utils/version" - "github.com/apex/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -25,7 +25,7 @@ var versionCmd = &cobra.Command{ gitCommit := viper.GetString("gitCommit") buildDate := viper.GetString("buildDate") - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "Version": viper.GetString("version"), "Branch": gitBranch, "Commit": gitCommit, diff --git a/ttnctl/cmd/version_homebrew.go b/ttnctl/cmd/version_homebrew.go index b029d4edb..8c201c1a3 100644 --- a/ttnctl/cmd/version_homebrew.go +++ b/ttnctl/cmd/version_homebrew.go @@ -6,7 +6,7 @@ package cmd import ( - "github.com/apex/log" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -18,7 +18,7 @@ var versionCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { gitCommit := viper.GetString("gitCommit") buildDate := viper.GetString("buildDate") - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "Version": viper.GetString("version") + "-homebrew", "Commit": gitCommit, "BuildDate": buildDate, diff --git a/ttnctl/util/account.go b/ttnctl/util/account.go index 723a26162..ac8ec4c9d 100644 --- a/ttnctl/util/account.go +++ b/ttnctl/util/account.go @@ -16,7 +16,7 @@ import ( "github.com/TheThingsNetwork/go-account-lib/cache" "github.com/TheThingsNetwork/go-account-lib/oauth" "github.com/TheThingsNetwork/go-account-lib/tokens" - "github.com/apex/log" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/spf13/viper" "golang.org/x/oauth2" ) @@ -74,7 +74,7 @@ func getAccountServerTokenSource(token *oauth2.Token) oauth2.TokenSource { return getOAuth().TokenSource(token) } -func getStoredToken(ctx log.Interface) *oauth2.Token { +func getStoredToken(ctx ttnlog.Interface) *oauth2.Token { tokenCache := GetTokenCache() data, err := tokenCache.Get(tokenName()) if err != nil { @@ -92,7 +92,7 @@ func getStoredToken(ctx log.Interface) *oauth2.Token { return token } -func saveToken(ctx log.Interface, token *oauth2.Token) { +func saveToken(ctx ttnlog.Interface, token *oauth2.Token) { data, err := json.Marshal(token) if err != nil { ctx.WithError(err).Fatal("Could not save access token") @@ -104,7 +104,7 @@ func saveToken(ctx log.Interface, token *oauth2.Token) { } type ttnctlTokenSource struct { - ctx log.Interface + ctx ttnlog.Interface source oauth2.TokenSource } @@ -118,7 +118,7 @@ func (s *ttnctlTokenSource) Token() (*oauth2.Token, error) { } // ForceRefreshToken forces a refresh of the access token -func ForceRefreshToken(ctx log.Interface) { +func ForceRefreshToken(ctx ttnlog.Interface) { tokenSource := GetTokenSource(ctx).(*ttnctlTokenSource) token, err := tokenSource.Token() if err != nil { @@ -130,7 +130,7 @@ func ForceRefreshToken(ctx log.Interface) { } // GetTokenSource builds a new oauth2.TokenSource that uses the ttnctl config to store the token -func GetTokenSource(ctx log.Interface) oauth2.TokenSource { +func GetTokenSource(ctx ttnlog.Interface) oauth2.TokenSource { if tokenSource != nil { return tokenSource } @@ -146,7 +146,7 @@ func GetTokenManager(accessToken string) tokens.Manager { } // GetAccount gets a new Account server client for ttnctl -func GetAccount(ctx log.Interface) *account.Account { +func GetAccount(ctx ttnlog.Interface) *account.Account { token, err := GetTokenSource(ctx).Token() if err != nil { ctx.WithError(err).Fatal("Could not get access token") @@ -159,7 +159,7 @@ func GetAccount(ctx log.Interface) *account.Account { } // Login does a login to the Account server with the given username and password -func Login(ctx log.Interface, code string) (*oauth2.Token, error) { +func Login(ctx ttnlog.Interface, code string) (*oauth2.Token, error) { config := getOAuth() token, err := config.Exchange(code) if err != nil { @@ -169,7 +169,7 @@ func Login(ctx log.Interface, code string) (*oauth2.Token, error) { return token, nil } -func TokenForScope(ctx log.Interface, scope string) string { +func TokenForScope(ctx ttnlog.Interface, scope string) string { token, err := GetTokenSource(ctx).Token() if err != nil { ctx.WithError(err).Fatal("Could not get token") diff --git a/ttnctl/util/cli_input.go b/ttnctl/util/cli_input.go new file mode 100644 index 000000000..9859a3d15 --- /dev/null +++ b/ttnctl/util/cli_input.go @@ -0,0 +1,112 @@ +package util + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "os" + "strconv" + "strings" + + "github.com/TheThingsNetwork/ttn/core/types" +) + +var stdin = bufio.NewReader(os.Stdin) +var ErrInvalidPayload = errors.New("Invalid payload (Should be hexadecimal)") + +func IsErrOutOfRange(err error) bool { + if ok := strings.HasSuffix(err.Error(), "value out of range"); !ok { + return false + } + return true +} + +func ReadPort() (uint8, error) { + fmt.Print("Port: ") + input, err := ReadInput(stdin) + if err != nil { + return 0, err + } + + port, err := parsePort(input) + if err != nil { + return 0, err + } + + return port, nil +} + +func parsePort(input string) (uint8, error) { + port, err := strconv.ParseUint(input, 10, 8) + if err != nil { + if IsErrOutOfRange(err) { + return 0, fmt.Errorf(fmt.Sprintf("The port number is too big (Should be uint8): %s", err.Error())) + } + return 0, err + } + + return uint8(port), nil +} + +func ReadFields() (map[string]interface{}, error) { + fmt.Print("Fields: ") + input, err := ReadInput(stdin) + if err != nil { + return nil, err + } + + parsedFields, err := parseFields(input) + if err != nil { + return nil, err + } + + return parsedFields, nil +} + +func parseFields(input string) (map[string]interface{}, error) { + parsedFields := make(map[string]interface{}) + err := json.Unmarshal([]byte(input), &parsedFields) + if err != nil { + return nil, err + } + + return parsedFields, nil +} + +func ReadPayload() ([]byte, error) { + fmt.Print("Payload: ") + input, err := ReadInput(stdin) + if err != nil { + return nil, err + } + + payload, err := parsePayload(input) + if err != nil { + return nil, err + } + + return payload, nil +} + +func parsePayload(input string) ([]byte, error) { + if len(input)%2 != 0 { + return nil, ErrInvalidPayload + } + + payload, err := types.ParseHEX(input, len(input)/2) + if err != nil { + return nil, err + } + + return payload, nil +} + +// ReadInput allows to read an input line from the user +func ReadInput(reader *bufio.Reader) (string, error) { + input, err := reader.ReadString('\n') + if err != nil { + return "", err + } + return strings.TrimRight(input, "\n"), nil +} diff --git a/ttnctl/util/cli_input_test.go b/ttnctl/util/cli_input_test.go new file mode 100644 index 000000000..c2bacc23a --- /dev/null +++ b/ttnctl/util/cli_input_test.go @@ -0,0 +1,78 @@ +package util + +import ( + "errors" + "testing" + + . "github.com/smartystreets/assertions" +) + +func TestIsErrOutOfRange(t *testing.T) { + a := New(t) + + err := errors.New("Dummy error") + ok := IsErrOutOfRange(err) + a.So(ok, ShouldBeFalse) + + err = errors.New("value out of range") + ok = IsErrOutOfRange(err) + a.So(ok, ShouldBeTrue) + + err = errors.New("whatever message: value out of range") + ok = IsErrOutOfRange(err) + a.So(ok, ShouldBeTrue) +} + +func TestParsePort(t *testing.T) { + a := New(t) + + n, err := parsePort("121212") + a.So(err, ShouldNotBeNil) + a.So(n, ShouldEqual, 0) + + n, err = parsePort("-12") + a.So(err, ShouldNotBeNil) + a.So(n, ShouldEqual, 0) + + n, err = parsePort("test") + a.So(err, ShouldNotBeNil) + a.So(n, ShouldEqual, 0) + + n, err = parsePort("1") + a.So(err, ShouldBeNil) + a.So(n, ShouldEqual, 1) +} + +func TestParseFields(t *testing.T) { + a := New(t) + + f, err := parseFields("test") + a.So(err, ShouldNotBeNil) + a.So(f, ShouldBeNil) + + // Syntax error: Invalid JSON object + f, err = parseFields(`{ "test": "value"`) + a.So(err, ShouldNotBeNil) + a.So(f, ShouldBeNil) + + f, err = parseFields(`{ "test": "value"}`) + a.So(err, ShouldBeNil) + a.So(f, ShouldResemble, map[string]interface{}{"test": "value"}) +} + +func TestParsePayload(t *testing.T) { + a := New(t) + + // Invalid input: Not hexadecimal + p, err := parsePayload("test") + a.So(err, ShouldNotBeNil) + a.So(p, ShouldBeNil) + + // Invalid input: not hexadecimal + p, err = parsePayload("123") + a.So(err, ShouldEqual, ErrInvalidPayload) + a.So(p, ShouldBeNil) + + p, err = parsePayload("1234") + a.So(err, ShouldBeNil) +} diff --git a/ttnctl/util/config.go b/ttnctl/util/config.go index 461f5714b..7b5f741eb 100644 --- a/ttnctl/util/config.go +++ b/ttnctl/util/config.go @@ -11,8 +11,8 @@ import ( yaml "gopkg.in/yaml.v2" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/core/types" - "github.com/apex/log" "github.com/spf13/viper" ) @@ -138,7 +138,7 @@ func setData(file, key string, data interface{}) error { } // GetAppEUI returns the AppEUI that must be set in the command options or config -func GetAppEUI(ctx log.Interface) types.AppEUI { +func GetAppEUI(ctx ttnlog.Interface) types.AppEUI { appEUIString := viper.GetString("app-eui") if appEUIString == "" { appData := readData(appFilename) @@ -161,7 +161,7 @@ func GetAppEUI(ctx log.Interface) types.AppEUI { } // SetApp stores the app EUI preference -func SetAppEUI(ctx log.Interface, appEUI types.AppEUI) { +func SetAppEUI(ctx ttnlog.Interface, appEUI types.AppEUI) { err := setData(appFilename, euiKey, appEUI.String()) if err != nil { ctx.WithError(err).Fatal("Could not save app EUI") @@ -169,7 +169,7 @@ func SetAppEUI(ctx log.Interface, appEUI types.AppEUI) { } // GetAppID returns the AppID that must be set in the command options or config -func GetAppID(ctx log.Interface) string { +func GetAppID(ctx ttnlog.Interface) string { appID := viper.GetString("app-id") if appID == "" { appData := readData(appFilename) @@ -187,7 +187,7 @@ func GetAppID(ctx log.Interface) string { } // SetApp stores the app ID and app EUI preferences -func SetApp(ctx log.Interface, appID string, appEUI types.AppEUI) { +func SetApp(ctx ttnlog.Interface, appID string, appEUI types.AppEUI) { config := readData(appFilename) config[idKey] = appID config[euiKey] = appEUI.String() diff --git a/ttnctl/util/context.go b/ttnctl/util/context.go index 2a72e0748..00b16d5ba 100644 --- a/ttnctl/util/context.go +++ b/ttnctl/util/context.go @@ -8,7 +8,7 @@ import ( "os" "os/user" - "github.com/apex/log" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/spf13/viper" "golang.org/x/net/context" // See https://github.com/grpc/grpc-go/issues/711" "google.golang.org/grpc/metadata" @@ -27,7 +27,7 @@ func GetID() string { } // GetContext returns a new context -func GetContext(ctx log.Interface, extraPairs ...string) context.Context { +func GetContext(ctx ttnlog.Interface, extraPairs ...string) context.Context { token, err := GetTokenSource(ctx).Token() if err != nil { ctx.WithError(err).Fatal("Could not get token") diff --git a/ttnctl/util/discovery.go b/ttnctl/util/discovery.go index 677a110b7..f54baa100 100644 --- a/ttnctl/util/discovery.go +++ b/ttnctl/util/discovery.go @@ -7,15 +7,15 @@ import ( "io/ioutil" "path" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/api" "github.com/TheThingsNetwork/ttn/api/discovery" - "github.com/apex/log" "github.com/spf13/viper" "google.golang.org/grpc" ) // GetDiscovery gets the Discovery client for ttnctl -func GetDiscovery(ctx log.Interface) (*grpc.ClientConn, discovery.DiscoveryClient) { +func GetDiscovery(ctx ttnlog.Interface) (*grpc.ClientConn, discovery.DiscoveryClient) { path := path.Join(GetDataDir(), "/ca.cert") cert, err := ioutil.ReadFile(path) if err == nil && !api.RootCAs.AppendCertsFromPEM(cert) { diff --git a/ttnctl/util/handler.go b/ttnctl/util/handler.go index bd162c332..879d07f5c 100644 --- a/ttnctl/util/handler.go +++ b/ttnctl/util/handler.go @@ -5,16 +5,16 @@ package util import ( "github.com/TheThingsNetwork/go-account-lib/scope" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/api/discovery" "github.com/TheThingsNetwork/ttn/api/handler" "github.com/TheThingsNetwork/ttn/utils/errors" - "github.com/apex/log" "github.com/spf13/viper" "google.golang.org/grpc" ) // GetHandlerManager gets a new HandlerManager for ttnctl -func GetHandlerManager(ctx log.Interface, appID string) (*grpc.ClientConn, *handler.ManagerClient) { +func GetHandlerManager(ctx ttnlog.Interface, appID string) (*grpc.ClientConn, *handler.ManagerClient) { ctx.WithField("Handler", viper.GetString("handler-id")).Info("Discovering Handler...") dscConn, client := GetDiscovery(ctx) defer dscConn.Close() diff --git a/ttnctl/util/mqtt.go b/ttnctl/util/mqtt.go index cd3bb3475..e36fed3f0 100644 --- a/ttnctl/util/mqtt.go +++ b/ttnctl/util/mqtt.go @@ -8,15 +8,14 @@ import ( "fmt" "strings" - "github.com/TheThingsNetwork/go-utils/log/apex" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/mqtt" - "github.com/apex/log" "github.com/gosuri/uitable" "github.com/spf13/viper" ) // GetMQTT connects a new MQTT clients with the specified credentials -func GetMQTT(ctx log.Interface) mqtt.Client { +func GetMQTT(ctx ttnlog.Interface) mqtt.Client { username, password, err := getMQTTCredentials(ctx) if err != nil { ctx.WithError(err).Fatal("Failed to get MQTT credentials") @@ -28,9 +27,9 @@ func GetMQTT(ctx log.Interface) mqtt.Client { ctx.Fatal("TLS connections are not yet supported by ttnctl") } broker := fmt.Sprintf("%s://%s", mqttProto, viper.GetString("mqtt-address")) - client := mqtt.NewClient(apex.Wrap(ctx), "ttnctl", username, password, broker) + client := mqtt.NewClient(ctx, "ttnctl", username, password, broker) - ctx.WithFields(log.Fields{ + ctx.WithFields(ttnlog.Fields{ "MQTT Broker": broker, "Username": username, }).Info("Connecting to MQTT...") @@ -42,7 +41,7 @@ func GetMQTT(ctx log.Interface) mqtt.Client { return client } -func getMQTTCredentials(ctx log.Interface) (username string, password string, err error) { +func getMQTTCredentials(ctx ttnlog.Interface) (username string, password string, err error) { username = viper.GetString("mqtt-username") password = viper.GetString("mqtt-password") if username != "" { @@ -57,7 +56,7 @@ func getMQTTCredentials(ctx log.Interface) (username string, password string, er return getAppMQTTCredentials(ctx) } -func getAppMQTTCredentials(ctx log.Interface) (string, string, error) { +func getAppMQTTCredentials(ctx ttnlog.Interface) (string, string, error) { appID := GetAppID(ctx) account := GetAccount(ctx) diff --git a/ttnctl/util/print_config.go b/ttnctl/util/print_config.go index 32e5a4527..3a99f41d5 100644 --- a/ttnctl/util/print_config.go +++ b/ttnctl/util/print_config.go @@ -6,11 +6,11 @@ package util import ( "fmt" - "github.com/apex/log" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/spf13/viper" ) -func PrintConfig(ctx log.Interface, debug bool) { +func PrintConfig(ctx ttnlog.Interface, debug bool) { prt := ctx.Infof if debug { prt = ctx.Debugf diff --git a/ttnctl/util/router.go b/ttnctl/util/router.go index a9251280d..c38a07bab 100644 --- a/ttnctl/util/router.go +++ b/ttnctl/util/router.go @@ -4,16 +4,16 @@ package util import ( + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/TheThingsNetwork/ttn/api/discovery" "github.com/TheThingsNetwork/ttn/api/router" "github.com/TheThingsNetwork/ttn/utils/errors" - "github.com/apex/log" "github.com/spf13/viper" "google.golang.org/grpc" ) // GetRouter starts a connection with the router -func GetRouter(ctx log.Interface) (*grpc.ClientConn, router.RouterClient) { +func GetRouter(ctx ttnlog.Interface) (*grpc.ClientConn, router.RouterClient) { ctx.Info("Discovering Router...") dscConn, client := GetDiscovery(ctx) defer dscConn.Close() @@ -32,7 +32,7 @@ func GetRouter(ctx log.Interface) (*grpc.ClientConn, router.RouterClient) { } // GetRouterManager starts a management connection with the router -func GetRouterManager(ctx log.Interface) (*grpc.ClientConn, router.RouterManagerClient) { +func GetRouterManager(ctx ttnlog.Interface) (*grpc.ClientConn, router.RouterManagerClient) { ctx.Info("Discovering Router...") dscConn, client := GetDiscovery(ctx) defer dscConn.Close() diff --git a/utils/testing/testing.go b/utils/testing/testing.go index aff2b9493..60bb507cf 100644 --- a/utils/testing/testing.go +++ b/utils/testing/testing.go @@ -11,15 +11,18 @@ import ( "testing" "time" - "github.com/apex/log" + ttnlog "github.com/TheThingsNetwork/go-utils/log" + ttnapex "github.com/TheThingsNetwork/go-utils/log/apex" + apexlog "github.com/apex/log" ) -func GetLogger(t *testing.T, tag string) log.Interface { - logger := &log.Logger{ +func GetLogger(t *testing.T, tag string) ttnlog.Interface { + logger := &apexlog.Logger{ Handler: NewLogHandler(t), - Level: log.DebugLevel, + Level: apexlog.DebugLevel, } - return logger.WithField("tag", tag) + ctx := logger.WithField("tag", tag) + return ttnapex.Wrap(ctx) } // WaitGroup is an extension of sync.WaitGroup with a WaitFor function for testing diff --git a/utils/version/version.go b/utils/version/version.go index 40324c241..20fcfa805 100644 --- a/utils/version/version.go +++ b/utils/version/version.go @@ -17,7 +17,7 @@ import ( "strings" "time" - "github.com/apex/log" + ttnlog "github.com/TheThingsNetwork/go-utils/log" "github.com/kardianos/osext" "github.com/spf13/viper" ) @@ -117,7 +117,7 @@ func GetLatest(binary string) ([]byte, error) { } // Selfupdate runs a self-update for the current binary -func Selfupdate(ctx log.Interface, component string) { +func Selfupdate(ctx ttnlog.Interface, component string) { if viper.GetString("gitBranch") == "unknown" { ctx.Infof("You are not using an official %s build. Not proceeding with the update", component) return diff --git a/vendor/vendor.json b/vendor/vendor.json index 7e8c1e2ad..27524ecb1 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -74,35 +74,47 @@ "revision": "357d32d3e59e60cf7fabd600e51b24988a29d770", "revisionTime": "2017-01-19T09:18:16Z" }, + { + "checksumSHA1": "FCeptJpMzjDNGH+vammILeR7MsI=", + "path": "github.com/TheThingsNetwork/go-utils/encoding", + "revision": "e15b5ceb87979c39b9170a3ec1864fec09e2e392", + "revisionTime": "2017-01-27T08:51:05Z" + }, + { + "checksumSHA1": "r2PGxRMFTm0RQ2xLFN5epifTbOM=", + "path": "github.com/TheThingsNetwork/go-utils/grpc/interceptor", + "revision": "e15b5ceb87979c39b9170a3ec1864fec09e2e392", + "revisionTime": "2017-01-27T08:51:05Z" + }, { "checksumSHA1": "T7iFQUlCUAv4cJNDZC0//46Nbio=", "path": "github.com/TheThingsNetwork/go-utils/handlers/cli", - "revision": "7f83485e15fda6a26871c4c68adb32d4a36c9b53", - "revisionTime": "2017-01-17T13:07:22Z" + "revision": "e15b5ceb87979c39b9170a3ec1864fec09e2e392", + "revisionTime": "2017-01-27T08:51:05Z" }, { "checksumSHA1": "aXt7ZSqIfsHWBbJPgHFjqtyxyQ0=", "path": "github.com/TheThingsNetwork/go-utils/log", - "revision": "7f83485e15fda6a26871c4c68adb32d4a36c9b53", - "revisionTime": "2017-01-17T13:07:22Z" + "revision": "e15b5ceb87979c39b9170a3ec1864fec09e2e392", + "revisionTime": "2017-01-27T08:51:05Z" }, { "checksumSHA1": "RdI5upcV6MHSjr5Y9zogYvbeURw=", "path": "github.com/TheThingsNetwork/go-utils/log/apex", - "revision": "7f83485e15fda6a26871c4c68adb32d4a36c9b53", - "revisionTime": "2017-01-17T13:07:22Z" + "revision": "e15b5ceb87979c39b9170a3ec1864fec09e2e392", + "revisionTime": "2017-01-27T08:51:05Z" }, { "checksumSHA1": "sQ0vy3MCGY1WgK9xldn1V6pMeZk=", "path": "github.com/TheThingsNetwork/go-utils/log/grpc", - "revision": "7f83485e15fda6a26871c4c68adb32d4a36c9b53", - "revisionTime": "2017-01-17T13:07:22Z" + "revision": "e15b5ceb87979c39b9170a3ec1864fec09e2e392", + "revisionTime": "2017-01-27T08:51:05Z" }, { "checksumSHA1": "kLFTtAVcjZbHXybodGAqJ8wxflY=", "path": "github.com/TheThingsNetwork/go-utils/roots", - "revision": "7f83485e15fda6a26871c4c68adb32d4a36c9b53", - "revisionTime": "2017-01-17T13:07:22Z" + "revision": "e15b5ceb87979c39b9170a3ec1864fec09e2e392", + "revisionTime": "2017-01-27T08:51:05Z" }, { "checksumSHA1": "nV/tpqju2PSPowf3Pq2RCOUESFY=", @@ -201,7 +213,7 @@ "revisionTime": "2017-01-18T11:17:54Z" }, { - "checksumSHA1": "XwDm1VMyZA/MLAK4Fst0yI7lyCE=", + "checksumSHA1": "6ZxSmrIx3Jd15aou16oG0HPylP4=", "path": "github.com/gogo/protobuf/proto", "revision": "6e505820968158ec7ac58ea92bba501a82b41171", "revisionTime": "2017-01-18T11:17:54Z" @@ -225,7 +237,7 @@ "revisionTime": "2016-11-17T03:31:26Z" }, { - "checksumSHA1": "q3Bc7JpLWBqhZ4M7oreGo34RSkc=", + "checksumSHA1": "kBeNcaKk56FguvPSUCEaH6AxpRc=", "path": "github.com/golang/protobuf/proto", "revision": "8ee79997227bf9b34611aee7946ae64735e6fd93", "revisionTime": "2016-11-17T03:31:26Z" @@ -675,7 +687,7 @@ "revisionTime": "2017-01-14T04:22:49Z" }, { - "checksumSHA1": "hyK05cmzm+vPH1OO+F1AkvES3sw=", + "checksumSHA1": "XH7CgbL5Z8COUc+MKrYqS3FFosY=", "path": "golang.org/x/oauth2", "revision": "314dd2c0bf3ebd592ec0d20847d27e79d0dbe8dd", "revisionTime": "2016-12-14T09:25:55Z" @@ -789,7 +801,7 @@ "revisionTime": "2017-01-16T08:01:07Z" }, { - "checksumSHA1": "M1xjRvZ0RQod9w9UcIttyk/4EuU=", + "checksumSHA1": "MQyhe1N+NO0+uyJiEc6M2WGfb3s=", "path": "gopkg.in/redis.v5/internal", "revision": "8829ddcd8bdb333e477cc845946c4b9b2ef66280", "revisionTime": "2017-01-16T08:01:07Z"