From 636e9c180adf565928c4a20e88c73d4f97794dad Mon Sep 17 00:00:00 2001 From: liang chenye Date: Thu, 25 Aug 2016 18:23:46 +0800 Subject: [PATCH] add snapshot service to implement scan feature Signed-off-by: liang chenye --- handler/appv1.go | 65 ++++++++- models/appv1.go | 6 + models/hook.go | 86 ++++++++---- router/router.go | 5 +- tests/unit/testdata/testmd5 | 1 + .../unit/updateservice_snapshot_appv1_test.go | 110 +++++++++++++++ .../updateservice_snapshot_snapshot_test.go | 125 ++++++++++++++++++ updateservice/snapshot/appv1.go | 67 ++++++++++ updateservice/snapshot/snapshot.go | 114 ++++++++++++++++ 9 files changed, 547 insertions(+), 32 deletions(-) create mode 100644 tests/unit/testdata/testmd5 create mode 100644 tests/unit/updateservice_snapshot_appv1_test.go create mode 100644 tests/unit/updateservice_snapshot_snapshot_test.go create mode 100644 updateservice/snapshot/appv1.go create mode 100644 updateservice/snapshot/snapshot.go diff --git a/handler/appv1.go b/handler/appv1.go index 396102d..565b90c 100644 --- a/handler/appv1.go +++ b/handler/appv1.go @@ -283,7 +283,9 @@ func AppDeleteFileV1Handler(ctx *macaron.Context) (int, []byte) { return httpRet("AppV1 Delete data", nil, err) } -func AppRegistScanHooksHandler(ctx *macaron.Context) (int, []byte) { +// AppRegistScanHooksV1Handler adds a scan plugin to a user repo +// TODO: to make it easier as a start, we assume each repo could only have one scan plugin +func AppRegistScanHooksV1Handler(ctx *macaron.Context) (int, []byte) { data, err := ctx.Req.Body().Bytes() if err != nil { log.Errorf("[%s] Req.Body.Bytes error: %s", ctx.Req.RequestURI, err.Error()) @@ -292,8 +294,11 @@ func AppRegistScanHooksHandler(ctx *macaron.Context) (int, []byte) { return http.StatusBadRequest, result } - var reg models.ScanHookRegist - err = json.Unmarshal(data, ®) + type scanPlugin struct { + Name string + } + var n scanPlugin + err = json.Unmarshal(data, &n) if err != nil { log.Errorf("[%s] Invalid body data: %s", ctx.Req.RequestURI, err.Error()) @@ -301,9 +306,10 @@ func AppRegistScanHooksHandler(ctx *macaron.Context) (int, []byte) { return http.StatusBadRequest, result } + var reg models.ScanHookRegist namespace := ctx.Params(":namespace") repository := ctx.Params(":repository") - err = reg.Regist(namespace, repository, reg.ImageName) + err = reg.Regist("appv1", namespace, repository, n.Name) if err != nil { log.Errorf("[%s] scan hook regist error: %s", ctx.Req.RequestURI, err.Error()) @@ -314,8 +320,8 @@ func AppRegistScanHooksHandler(ctx *macaron.Context) (int, []byte) { return httpRet("AppV1 Scan Hook Regist", nil, err) } -// AppCallbackScanHooksHandler gets callback from container and save the scan result. -func AppCallbackScanHooksHandler(ctx *macaron.Context) (int, []byte) { +// AppCallbackScanHooksV1Handler gets callback from container and save the scan result. +func AppCallbackScanHooksV1Handler(ctx *macaron.Context) (int, []byte) { data, err := ctx.Req.Body().Bytes() if err != nil { log.Errorf("[%s] Req.Body.Bytes error: %s", ctx.Req.RequestURI, err.Error()) @@ -336,3 +342,50 @@ func AppCallbackScanHooksHandler(ctx *macaron.Context) (int, []byte) { return httpRet("AppV1 Scan Hook Callback", nil, err) } + +// AppActiveScanHooksTaskV1Handler actives a scan task +func AppActiveScanHooksTaskV1Handler(ctx *macaron.Context) (int, []byte) { + namespace := ctx.Params(":namespace") + repository := ctx.Params(":repository") + + var r models.ScanHookRegist + rID, err := r.FindID("appv1", namespace, repository) + if err != nil { + log.Errorf("[%s] scan hook callback error: %s", ctx.Req.RequestURI, err.Error()) + + result, _ := json.Marshal(map[string]string{"Error": "Donnot have registed scan plugin"}) + return http.StatusBadRequest, result + } + + a := models.ArtifactV1{ + OS: ctx.Params(":os"), + Arch: ctx.Params(":arch"), + App: ctx.Params(":app"), + Tag: ctx.Params(":tag"), + } + a, err = a.Get() + if err != nil { + log.Errorf("[%s] scan hook callback error: %s", ctx.Req.RequestURI, err.Error()) + + result, _ := json.Marshal(map[string]string{"Error": "Cannot find artifactv1"}) + return http.StatusBadRequest, result + } + + // create a task + var t models.ScanHookTask + tID, err := t.Put(rID, a.Path) + if err != nil { + log.Errorf("[%s] scan hook callback error: %s", ctx.Req.RequestURI, err.Error()) + + result, _ := json.Marshal(map[string]string{"Error": "Fail to create a scan task"}) + return http.StatusBadRequest, result + } + + idBytes, err := utils.TokenMarshal(tID, setting.ScanKey) + + val := struct { + TaskID string + }{TaskID: string(idBytes)} + + return httpRet("AppV1 Active Scan Hook Task", val, nil) +} diff --git a/models/appv1.go b/models/appv1.go index 6487009..369d9cd 100644 --- a/models/appv1.go +++ b/models/appv1.go @@ -98,6 +98,12 @@ func (app *AppV1) Delete(artifact ArtifactV1) error { return nil } +// Get gets full info by os/arch/app/tag +func (a *ArtifactV1) Get() (ArtifactV1, error) { + // TODO + return *a, nil +} + func (a *ArtifactV1) GetName() string { if ok, _ := a.isValid(); !ok { return "" diff --git a/models/hook.go b/models/hook.go index d8de9a7..207de8b 100644 --- a/models/hook.go +++ b/models/hook.go @@ -18,48 +18,66 @@ package models import ( "errors" + "strconv" "time" "github.com/containerops/dockyard/setting" + "github.com/containerops/dockyard/updateservice/snapshot" "github.com/containerops/dockyard/utils" ) // ScanHookRegist: // Namespace/Repository contains images/apps/vms to be scaned -// ImageName is used to scan the images/apps/vms within a repository -// NOTE: no need to record the type of a repository ("docker/v1", "app/v1"), -// a user should regist his/her repository with the right ImageName +// ScanPluginName is a plugin name of 'snapshot' type ScanHookRegist struct { - ID int64 `json:"id" gorm:"primary_key"` - Namespace string `json:"namespace" sql:"not null;type:varchar(255)"` - Repository string `json:"repository" sql:"not null;type:varchar(255)"` - ImageName string `json:"ImageName" sql:"not null;type:varchar(255)"` + ID int64 `json:"id" gorm:"primary_key"` + Proto string `json:"proto" sql:"not null;type:varchar(255)"` + Namespace string `json:"namespace" sql:"not null;type:varchar(255)"` + Repository string `json:"repository" sql:"not null;type:varchar(255)"` + ScanPluginName string `json:"scanPluginName" sql:"not null;type:varchar(255)"` } // Regist regists a repository with a scan image -// A namespace/repository could have multiple ImageNames, -// but should not have multiple records with same Namespace&Repository&ImageName. -func (s *ScanHookRegist) Regist(n, r, image string) error { - if n == "" || r == "" || image == "" { - return errors.New("'Namespace', 'Repository' and 'ImageName' should not be empty") +// A namespace/repository could have multiple ScanPluginName, +// but now we only support one. +func (s *ScanHookRegist) Regist(p, n, r, name string) error { + if p == "" || n == "" || r == "" || name == "" { + return errors.New("'Proto', 'Namespace', 'Repository' and 'ScanPluginName' should not be empty") } - s.Namespace, s.Repository, s.ImageName = n, r, image + + if ok, err := snapshot.IsSnapshotSupported(p, name); !ok { + return err + } + + s.Proto, s.Namespace, s.Repository, s.ScanPluginName = p, n, r, name //TODO: add to db return nil } // UnRegist unregists a repository with a scan image // if ImageName is nil, unregist all the scan images. -func (s *ScanHookRegist) UnRegist(n, r string) error { - if n == "" || r == "" { - return errors.New("'Namespace', 'Repository' should not be empty") +func (s *ScanHookRegist) UnRegist(p, n, r string) error { + if p == "" || n == "" || r == "" { + return errors.New("'Proto', 'Namespace', 'Repository' should not be empty") } - s.Namespace, s.Repository = n, r + s.Proto, s.Namespace, s.Repository = p, n, r //TODO: remove from db return nil } +// FindByID finds content by id +func (s *ScanHookRegist) FindByID(id int64) (ScanHookRegist, error) { + //TODO: query db + return *s, nil +} + +// FindID finds id by Proto, Namespace and Repository +func (s *ScanHookRegist) FindID(p, n, r string) (int64, error) { + //TODO: query db + return 0, nil +} + // ListScanHooks returns a list of registed scan hooks of a repository func (s *ScanHookRegist) List(n, r string) ([]ScanHookRegist, error) { if n == "" || r == "" { @@ -70,7 +88,8 @@ func (s *ScanHookRegist) List(n, r string) ([]ScanHookRegist, error) { // ScanHookTask is the scan task type ScanHookTask struct { - ID int64 `json:"id" gorm:"primary_key"` + ID int64 `json:"id" gorm:"primary_key"` + //Path is image url now Path string `json:"path" sql:"not null;type:varchar(255)"` Callback string `json:"callback" sql:"not null;type:varchar(255)"` // ID of ScanHookRegist @@ -82,12 +101,31 @@ type ScanHookTask struct { UpdatedAt time.Time `json:"update_at" sql:""` } -func (t *ScanHookTask) Put(p, c string, rID int64) error { - if p == "" || c == "" || rID == 0 { - return errors.New("'Namespace', 'Repository' and RegistID should not be empty") +// Put returns task id +func (t *ScanHookTask) Put(rID int64, url string) (int64, error) { + if url == "" || rID == 0 { + return 0, errors.New("'URL' and 'RegistID' should not be empty") } + //TODO: add to db and get task ID - return nil + var reg ScanHookRegist + reg, err := reg.FindByID(rID) + if err != nil { + return 0, err + } + + // Do the real scan work + s, err := snapshot.NewUpdateServiceSnapshot(reg.ScanPluginName, strconv.FormatInt(rID, 10), url, nil) + if err != nil { + return 0, err + } + + err = s.Process() + if err != nil { + return 0, err + } + + return 0, nil } func (t *ScanHookTask) Update(status string) error { @@ -97,14 +135,14 @@ func (t *ScanHookTask) Update(status string) error { func (t *ScanHookTask) Find(encodedCallbackID string) error { //TODO: update status and updatedAt - var id int + var id int64 err := utils.TokenUnmarshal(encodedCallbackID, setting.ScanKey, &id) return err } func (t *ScanHookTask) UpdateResult(encodedCallbackID string, data []byte) error { - var id int + var id int64 err := utils.TokenUnmarshal(encodedCallbackID, setting.ScanKey, &id) if err != nil { return err diff --git a/router/router.go b/router/router.go index 3fe7b14..23a58e5 100644 --- a/router/router.go +++ b/router/router.go @@ -114,8 +114,9 @@ func SetRouters(m *macaron.Macaron) { m.Delete("/:os/:arch/:app/?:tag", handler.AppDeleteFileV1Handler) //Content Scan - m.Post("/shook", handler.AppRegistScanHooksHandler) - m.Post("/shook/:callbackID", handler.AppCallbackScanHooksHandler) + m.Post("/shook", handler.AppRegistScanHooksV1Handler) + m.Post("/shook/:callbackID", handler.AppCallbackScanHooksV1Handler) + m.Post("/:os/:arch/:app/shook/?:tag", handler.AppActiveScanHooksTaskV1Handler) }) }) }) diff --git a/tests/unit/testdata/testmd5 b/tests/unit/testdata/testmd5 new file mode 100644 index 0000000..f3ced09 --- /dev/null +++ b/tests/unit/testdata/testmd5 @@ -0,0 +1 @@ +this is test md5. diff --git a/tests/unit/updateservice_snapshot_appv1_test.go b/tests/unit/updateservice_snapshot_appv1_test.go new file mode 100644 index 0000000..af5f66d --- /dev/null +++ b/tests/unit/updateservice_snapshot_appv1_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2016 The ContainerOps Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package unittest + +import ( + "errors" + "fmt" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/containerops/dockyard/updateservice/snapshot" +) + +func TestAppv1New(t *testing.T) { + cases := []struct { + id string + url string + expected bool + }{ + {"id", "url", true}, + {"", "url", false}, + {"id", "", false}, + } + + var appv1 snapshot.UpdateServiceSnapshotAppv1 + for _, c := range cases { + _, err := appv1.New(c.id, c.url, nil) + assert.Equal(t, c.expected, err == nil, "Fail to create new snapshot appv1") + } +} + +func TestAppv1Supported(t *testing.T) { + cases := []struct { + proto string + expected bool + }{ + {"appv1", true}, + {"invalid", false}, + } + + var appv1 snapshot.UpdateServiceSnapshotAppv1 + for _, c := range cases { + assert.Equal(t, c.expected, appv1.Supported(c.proto), "Fail to get supported status") + } +} + +var ( + cbMap = make(map[string]snapshot.UpdateServiceSnapshotOutput) +) + +func testCB(id string, data snapshot.UpdateServiceSnapshotOutput) error { + if id != "1" && id != "2" { + return errors.New("invalid id") + } + + cbMap[id] = data + return nil +} + +func TestAppv1Process(t *testing.T) { + for n, _ := range cbMap { + delete(cbMap, n) + } + + cases := []struct { + id string + url string + cb snapshot.Callback + pExpected bool + idExpected bool + md5 string + }{ + {"1", "testmd5", testCB, true, true, "ffe7c736f2aa54531ac6430e3cbf2545"}, + {"2", "invalid", testCB, true, true, ""}, + {"3", "testmd5", testCB, false, false, ""}, + {"4", "testmd5", nil, true, false, ""}, + } + + var appv1 snapshot.UpdateServiceSnapshotAppv1 + _, path, _, _ := runtime.Caller(0) + dir := filepath.Join(filepath.Dir(path), "testdata") + + for _, c := range cases { + a, _ := appv1.New(c.id, filepath.Join(dir, c.url), c.cb) + err := a.Process() + assert.Equal(t, c.pExpected, err == nil, "Fail to get correct process output") + + data, ok := cbMap[c.id] + assert.Equal(t, c.idExpected, ok, "Fail to call cb") + if ok { + assert.Equal(t, c.md5, fmt.Sprintf("%x", data.Data), "Fail to call cb md5") + } + } +} diff --git a/tests/unit/updateservice_snapshot_snapshot_test.go b/tests/unit/updateservice_snapshot_snapshot_test.go new file mode 100644 index 0000000..2384027 --- /dev/null +++ b/tests/unit/updateservice_snapshot_snapshot_test.go @@ -0,0 +1,125 @@ +/* +Copyright 2016 The ContainerOps Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package unittest + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/containerops/dockyard/updateservice/snapshot" +) + +type UpdateServiceSnapshotMock struct { +} + +func (m *UpdateServiceSnapshotMock) New(id, url string, callback snapshot.Callback) (snapshot.UpdateServiceSnapshot, error) { + return m, nil +} + +func (m *UpdateServiceSnapshotMock) Supported(proto string) bool { + return proto == "mock" +} + +func (m *UpdateServiceSnapshotMock) Process() error { + return nil +} + +func (m *UpdateServiceSnapshotMock) Description() string { + return "mock description" +} + +// expunge all the registed implementaions +func preTest() { + cases := []struct { + name string + f snapshot.UpdateServiceSnapshot + }{ + {"mname0", &UpdateServiceSnapshotMock{}}, + {"mname1", &UpdateServiceSnapshotMock{}}, + {"aname0", &snapshot.UpdateServiceSnapshotAppv1{}}, + {"aname1", &snapshot.UpdateServiceSnapshotAppv1{}}, + } + + snapshot.UnregisterAllSnapshot() + + for _, c := range cases { + snapshot.RegisterSnapshot(c.name, c.f) + } +} + +func TestRegisterSnapshot(t *testing.T) { + preTest() + + cases := []struct { + name string + f snapshot.UpdateServiceSnapshot + expected bool + }{ + {"", &UpdateServiceSnapshotMock{}, false}, + {"testsname", nil, false}, + {"testsname", &UpdateServiceSnapshotMock{}, true}, + {"testsname", &UpdateServiceSnapshotMock{}, false}, + } + + for _, c := range cases { + err := snapshot.RegisterSnapshot(c.name, c.f) + assert.Equal(t, c.expected, err == nil, "Fail to register snapshot") + } +} + +func TestListSnapshot(t *testing.T) { + preTest() + + strs := snapshot.ListSnapshotByProto("mock") + assert.Equal(t, 2, len(strs), "Fail to get correct snapshot list") +} + +func TestNewUpdateServiceSnapshot(t *testing.T) { + preTest() + + cases := []struct { + name string + expected bool + }{ + {"mname0", true}, + {"invalidname", false}, + } + + for _, c := range cases { + _, err := snapshot.NewUpdateServiceSnapshot(c.name, "id", "url", nil) + assert.Equal(t, c.expected, err == nil, "Fail to create new snapshot") + } +} + +func TestIsSnapshotSupported(t *testing.T) { + preTest() + + cases := []struct { + p string + n string + expected bool + }{ + {"mock", "mname0", true}, + {"mock", "invalid", false}, + {"invalid", "mname0", false}, + } + + for _, c := range cases { + ok, _ := snapshot.IsSnapshotSupported(c.p, c.n) + assert.Equal(t, c.expected, ok, "Fail to get supported result") + } +} diff --git a/updateservice/snapshot/appv1.go b/updateservice/snapshot/appv1.go new file mode 100644 index 0000000..cc6b343 --- /dev/null +++ b/updateservice/snapshot/appv1.go @@ -0,0 +1,67 @@ +/* +Copyright 2016 The ContainerOps Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package snapshot + +import ( + "crypto/md5" + "errors" + "io/ioutil" +) + +const ( + appv1Proto = "appv1" +) + +type UpdateServiceSnapshotAppv1 struct { + ID string + URL string + + callback Callback +} + +func (m *UpdateServiceSnapshotAppv1) New(id, url string, callback Callback) (UpdateServiceSnapshot, error) { + if id == "" || url == "" { + return nil, errors.New("id|url should not be empty") + } + + m.ID, m.URL, m.callback = id, url, callback + return m, nil +} + +func (m *UpdateServiceSnapshotAppv1) Supported(proto string) bool { + return proto == appv1Proto +} + +func (m *UpdateServiceSnapshotAppv1) Process() error { + var data UpdateServiceSnapshotOutput + + content, err := ioutil.ReadFile(m.URL) + if m.callback == nil { + return err + } + + if err == nil { + s := md5.Sum(content) + data.Data = s[:] + } + data.Error = err + + return m.callback(m.ID, data) +} + +func (m *UpdateServiceSnapshotAppv1) Description() string { + return "Scan the appv1 package, return its md5" +} diff --git a/updateservice/snapshot/snapshot.go b/updateservice/snapshot/snapshot.go new file mode 100644 index 0000000..0ac0616 --- /dev/null +++ b/updateservice/snapshot/snapshot.go @@ -0,0 +1,114 @@ +/* +Copyright 2016 The ContainerOps Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package snapshot + +import ( + "errors" + "fmt" + "sync" +) + +// TODO: +type UpdateServiceSnapshotOutput struct { + Data []byte + Error error +} + +// Callback is a function that a snapshot plugin use after finish the `Process` +// configuration. +type Callback func(id string, output UpdateServiceSnapshotOutput) error + +// UpdateServiceSnapshot represents the snapshot interface +type UpdateServiceSnapshot interface { + // `id` : callback id + // `url`: local file or local dir + // `callback`: if callback is nil, the caller could handle it by itself + // or the caller must implement calling this in `Process` + // TODO: we need to certify plugins.. + New(id, url string, callback Callback) (UpdateServiceSnapshot, error) + // `proto`: `appv1/dockerv1` for example + Supported(proto string) bool + Description() string + Process() error +} + +var ( + usSnapshotsLock sync.Mutex + usSnapshots = make(map[string]UpdateServiceSnapshot) +) + +// RegisterSnapshot provides a way to dynamically register an implementation of a +// snapshot type. +func RegisterSnapshot(name string, f UpdateServiceSnapshot) error { + if name == "" { + return errors.New("Could not register a Snapshot with an empty name") + } + if f == nil { + return errors.New("Could not register a nil Snapshot") + } + + usSnapshotsLock.Lock() + defer usSnapshotsLock.Unlock() + + if _, alreadyExists := usSnapshots[name]; alreadyExists { + return fmt.Errorf("Snapshot type '%s' is already registered", name) + } + usSnapshots[name] = f + return nil +} + +func UnregisterAllSnapshot() { + usSnapshotsLock.Lock() + defer usSnapshotsLock.Unlock() + + for n, _ := range usSnapshots { + delete(usSnapshots, n) + } +} + +func IsSnapshotSupported(proto, name string) (bool, error) { + f, ok := usSnapshots[name] + if !ok { + return false, fmt.Errorf("Cannot find plugin :%s", name) + } + + ok = f.Supported(proto) + if !ok { + return false, fmt.Errorf("Proto %s is not supported by plugin %s", proto, name) + } + + return true, nil +} + +func ListSnapshotByProto(proto string) (snapshots []string) { + for n, f := range usSnapshots { + if f.Supported(proto) { + snapshots = append(snapshots, n) + } + } + + return +} + +// NewUpdateServiceSnapshot creates a snapshot interface by a name and a url +func NewUpdateServiceSnapshot(name, id, url string, cb Callback) (UpdateServiceSnapshot, error) { + f, ok := usSnapshots[name] + if !ok { + return nil, fmt.Errorf("Snapshot '%s' not found", name) + } + return f.New(id, url, cb) +}