Skip to content

Commit

Permalink
feat: support load files from dir
Browse files Browse the repository at this point in the history
  • Loading branch information
@elonzh
elonzh committed Nov 6, 2020
1 parent 3010257 commit ff06066
Show file tree
Hide file tree
Showing 6 changed files with 172 additions and 103 deletions.
155 changes: 107 additions & 48 deletions cmd/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,58 +22,18 @@ THE SOFTWARE.
package cmd

import (
"os"

"github.com/elonzh/trumpet/transformers"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)

// configCmd represents the config command
var configCmd = &cobra.Command{
Use: "config",
Short: "",
Long: ``,
}

type Config struct {
LogLevel logrus.Level
Transformers map[string]*transformers.Transformer
}

var (
cfg = &Config{
LogLevel: logrus.InfoLevel,
Transformers: map[string]*transformers.Transformer{},
}
)

func registerTransformers(m map[string]string) {
for name, src := range m {
t, err := transformers.NewTransformer(name, src)
if err != nil {
logrus.WithError(err).WithField("Name", name).Fatalln("error when init Transformer")
}
if _, exists := cfg.Transformers[t.Name]; exists {
logrus.WithField("Name", name).Warnln("Transformer already exists")
}
cfg.Transformers[t.Name] = t
}
}

func init() {
builtinTransformers := map[string]string{
"feishu-to-dingtalk": `
def transform(request):
msg_type = request["body"]["msg_type"]
body = {}
if msg_type == "text":
body = {
"msgtype": "text",
"text": {"content": request["body"]["content"]["text"]},
}
request["body"] = body
return request
`,
"dingtalk-to-feishu": `
var BuiltinTransformers = []*transformers.Transformer{
{
FileName: "builtin/dingtalk-to-feishu.star",
Src: `
def transform(request):
msg_type = request["body"]["msgtype"]
body = {}
Expand All @@ -94,7 +54,106 @@ def transform(request):
request["body"] = body
return request
`,
},
{
FileName: "builtin/feishu-to-dingtalk.star",
Src: `
def transform(request):
msg_type = request["body"]["msg_type"]
body = {}
if msg_type == "text":
body = {
"msgtype": "text",
"text": {"content": request["body"]["content"]["text"]},
}
request["body"] = body
return request
`,
},
}

type Config struct {
LogLevel string
TransformersDir string
Transformers []*transformers.Transformer

allTransformers []*transformers.Transformer
m map[string]*transformers.Transformer
}

func (c *Config) LoadAllTransformers() {
allTransformers := make([]*transformers.Transformer, 0, len(BuiltinTransformers))
allTransformers = append(allTransformers, BuiltinTransformers...)
allTransformers = append(allTransformers, cfg.Transformers...)
if cfg.TransformersDir != "" {
trans, err := transformers.Load(cfg.TransformersDir)
if err != nil {
logrus.WithError(err).Fatalln()
}
allTransformers = append(allTransformers, trans...)
}
c.allTransformers = allTransformers

m := make(map[string]*transformers.Transformer, len(cfg.Transformers))
for _, t := range allTransformers {
if err := t.InitThread(); err != nil {
logrus.WithError(err).Fatalln()
}
if _, exists := m[t.Name]; exists {
logrus.WithField("Name", t.Name).Warnln("Transformer already exists")
}
m[t.Name] = t
}
c.m = m
}

func (c *Config) GetTransformer(name string) (*transformers.Transformer, bool) {
t, ok := c.m[name]
return t, ok
}

func initConfig() {
if cfgFile != "" {
viper.SetConfigFile(cfgFile)
} else {
viper.AddConfigPath(".")
viper.SetConfigName("config")
}
registerTransformers(builtinTransformers)
var err error
if err = viper.ReadInConfig(); os.IsNotExist(err) {
logrus.WithError(err).Fatalln()
}
logrus.WithField("ConfigFile", viper.ConfigFileUsed()).Infoln("read in config")
err = viper.Unmarshal(cfg)
if err != nil {
logrus.WithError(err).Fatalln("error when unmarshal config")
}
level, err := logrus.ParseLevel(cfg.LogLevel)
if err != nil {
logrus.WithError(err).Fatalln()
}
logrus.SetLevel(level)
if level >= logrus.DebugLevel {
logrus.WithField("Config", cfg).Debug()
}

cfg.LoadAllTransformers()
}

var (
cfg = &Config{
LogLevel: logrus.InfoLevel.String(),
}
)

// configCmd represents the config command
var configCmd = &cobra.Command{
Use: "config",
Short: "",
Long: ``,
}

func init() {
rootCmd.AddCommand(configCmd)
cobra.OnInitialize(initConfig)
}
28 changes: 0 additions & 28 deletions cmd/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@ THE SOFTWARE.
package cmd

import (
"os"

"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
Expand All @@ -44,36 +42,10 @@ func Execute() {
}

func init() {
cobra.OnInitialize(initConfig)

rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file (default is ./config.yaml)")
rootCmd.PersistentFlags().String("logLevel", "info", "")
err := viper.BindPFlag("logLevel", rootCmd.PersistentFlags().Lookup("logLevel"))
if err != nil {
panic(err)
}
}

func initConfig() {
if cfgFile != "" {
viper.SetConfigFile(cfgFile)
} else {
viper.AddConfigPath(".")
viper.SetConfigName("config")
}
var err error
if err = viper.ReadInConfig(); os.IsNotExist(err) {
logrus.WithError(err).Fatalln()
}
logrus.WithField("ConfigFile", viper.ConfigFileUsed()).Infoln("read in config")
cfg.LogLevel, err = logrus.ParseLevel(viper.GetString("logLevel"))
if err != nil {
logrus.WithError(err).Fatalln()
}
logrus.SetLevel(cfg.LogLevel)
if cfg.LogLevel >= logrus.DebugLevel {
logrus.WithField("Config", cfg).Debug()
}
logrus.SetFormatter(&logrus.TextFormatter{})
registerTransformers(viper.GetStringMapString("transformers"))
}
6 changes: 3 additions & 3 deletions cmd/serve.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ var proxy = httputil.ReverseProxy{
request.Header["X-Forwarded-For"] = nil
request.ContentLength = -1
delete(request.Header, "Content-Length")
if cfg.LogLevel >= logrus.DebugLevel {
if logrus.GetLevel() >= logrus.DebugLevel {
req, err := httputil.DumpRequest(request, true)
fmt.Printf(
"\n-------------------- Request --------------------\n%s\nDumpRequestError:%s\n",
Expand All @@ -33,7 +33,7 @@ var proxy = httputil.ReverseProxy{
}
},
ModifyResponse: func(response *http.Response) error {
if cfg.LogLevel >= logrus.DebugLevel {
if logrus.GetLevel() >= logrus.DebugLevel {
resp, err := httputil.DumpResponse(response, true)
fmt.Printf(
"\n-------------------- Request --------------------\n%s\nDumpResponseError:%s\n",
Expand Down Expand Up @@ -61,7 +61,7 @@ var serveCmd = &cobra.Command{
c.String(http.StatusBadRequest, err.Error())
return
}
transformer, ok := cfg.Transformers[transformerName]
transformer, ok := cfg.GetTransformer(transformerName)
if !ok {
c.String(http.StatusNotFound, "no such transformer `%s`", transformer)
return
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ require (
github.com/go-playground/validator/v10 v10.4.1 // indirect
github.com/golang/protobuf v1.4.3 // indirect
github.com/json-iterator/go v1.1.10 // indirect
github.com/sirupsen/logrus v1.2.0
github.com/sirupsen/logrus v1.7.0
github.com/spf13/cobra v1.1.1
github.com/spf13/viper v1.7.0
github.com/spf13/viper v1.7.1
github.com/ugorji/go v1.1.13 // indirect
go.starlark.net v0.0.0-20201014215153-dff0ae5b4820
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 // indirect
Expand Down
5 changes: 5 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
Expand All @@ -225,6 +227,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
Expand Down Expand Up @@ -313,6 +317,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down
77 changes: 55 additions & 22 deletions transformers/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"fmt"
"io/ioutil"
"net/http"
"path"
"path/filepath"
"regexp"
"strings"

Expand All @@ -14,6 +16,7 @@ import (
)

const (
FileSuffix = ".star"
transformFunctionName = "transform"
headerKey = starlark.String("header")
bodyKey = starlark.String("body")
Expand All @@ -36,13 +39,40 @@ func init() {

type Transformer struct {
Name string
src interface{}
FileName string
Src interface{}
thread *starlark.Thread
transFunc starlark.Value
}

func (t *Transformer) String() string {
return fmt.Sprintf("Transformer{Name: %s}", t.Name)
return fmt.Sprintf("Transformer{Name: %s, FileName: %s}", t.Name, t.FileName)
}

func (t *Transformer) InitThread() error {
if t.Name == "" {
t.Name = strings.TrimSuffix(filepath.Base(t.FileName), FileSuffix)
}
if err := validateName(t.Name); err != nil {
return err
}
t.thread = &starlark.Thread{
Name: t.Name,
}
predeclared := starlark.StringDict{
starlarkjson.Module.Name: starlarkjson.Module,
"struct": starlark.NewBuiltin("struct", starlarkstruct.Make),
}
globals, err := starlark.ExecFile(t.thread, t.FileName, t.Src, predeclared)
if err != nil {
return err
}
transFunc, ok := globals[transformFunctionName]
if !ok {
return fmt.Errorf("function `transformer` not found")
}
t.transFunc = transFunc
return nil
}

func (t *Transformer) requestToStarDict(req *http.Request) (*starlark.Dict, error) {
Expand Down Expand Up @@ -128,30 +158,33 @@ func validateName(s string) error {
return fmt.Errorf("%s is not a valid url slug as transformer name", s)
}

func NewTransformer(name string, src interface{}) (*Transformer, error) {
if err := validateName(name); err != nil {
func Load(dir string) ([]*Transformer, error) {
dir, err := filepath.Abs(dir)
if err != nil {
return nil, err
}
thread := &starlark.Thread{
Name: name,
}
predeclared := starlark.StringDict{
starlarkjson.Module.Name: starlarkjson.Module,
"struct": starlark.NewBuiltin("struct", starlarkstruct.Make),
}
globals, err := starlark.ExecFile(thread, name, src, predeclared)
files, err := ioutil.ReadDir(dir)
if err != nil {
return nil, err
}
transFunc, ok := globals[transformFunctionName]
if !ok {
return nil, fmt.Errorf("transformer not found")
}
t := &Transformer{
Name: name,
src: src,
thread: thread,
transFunc: transFunc,
rv := make([]*Transformer, 0, len(files))
for _, f := range files {
logEntry := logrus.WithFields(logrus.Fields{
"Dir": dir,
"FileName": f.Name(),
})

if f.IsDir() {
logEntry.Debugln("is a dir, ignore")
continue
}
if strings.HasSuffix(f.Name(), FileSuffix) {
filename := path.Join(dir, f.Name())
rv = append(rv, &Transformer{FileName: filename})
logEntry.Debugf("file loaded")
} else {
logEntry.Debugf("file name does not have suffix `%s`", FileSuffix)
}
}
return t, nil
return rv, nil
}

0 comments on commit ff06066

Please sign in to comment.