From bcaf87b57d1f8a00a7ab03530edf69836f21def8 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Fri, 5 Jul 2024 16:42:41 +0530 Subject: [PATCH 1/7] model Import command w/signoff Signed-off-by: Jougan-0 --- mesheryctl/internal/cli/root/model/import.go | 188 +++++++++++++++++++ mesheryctl/internal/cli/root/model/model.go | 2 +- server/handlers/component_handler.go | 138 ++++++++++++++ server/models/handlers.go | 2 +- server/router/server.go | 2 + 5 files changed, 330 insertions(+), 2 deletions(-) create mode 100644 mesheryctl/internal/cli/root/model/import.go diff --git a/mesheryctl/internal/cli/root/model/import.go b/mesheryctl/internal/cli/root/model/import.go new file mode 100644 index 00000000000..13628720587 --- /dev/null +++ b/mesheryctl/internal/cli/root/model/import.go @@ -0,0 +1,188 @@ +package model + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/layer5io/meshery/mesheryctl/internal/cli/root/config" + "github.com/layer5io/meshery/mesheryctl/pkg/utils" + "github.com/layer5io/meshery/server/models" + meshkitutils "github.com/layer5io/meshkit/utils" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +var importModelCmd = &cobra.Command{ + Use: "import", + Short: "import models from mesheryctl command", + Long: "import model by specifying the directory. Use 'import model --file [filepath]' to import models and register them to Meshery.", + Example: ` + import model --file /path/to/models + `, + PreRunE: func(cmd *cobra.Command, args []string) error { + // Check prerequisites + mctlCfg, err := config.GetMesheryCtl(viper.GetViper()) + if err != nil { + return err + } + err = utils.IsServerRunning(mctlCfg.GetBaseMesheryURL()) + if err != nil { + return err + } + ctx, err := mctlCfg.GetCurrentContext() + if err != nil { + return err + } + err = ctx.ValidateVersion() + if err != nil { + return err + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + dirpath, err := cmd.Flags().GetString("file") + if err != nil { + return err + } + + if dirpath == "" { + return fmt.Errorf("file path is required") + } + + tarData, err := compressDirectory(dirpath) + if err != nil { + return err + } + + fileName := filepath.Base(dirpath) + ".tar.gz" + err = sendTarToAPI(tarData, fileName) + if err != nil { + return err + } + + return nil + }, +} + +func compressDirectory(dirpath string) ([]byte, error) { + tw := meshkitutils.NewTarWriter() + defer tw.Close() + + err := filepath.Walk(dirpath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + fileData, err := io.ReadAll(file) + if err != nil { + return err + } + + relPath, err := filepath.Rel(filepath.Dir(dirpath), path) + if err != nil { + return err + } + + if err := tw.Compress(relPath, fileData); err != nil { + return err + } + + return nil + }) + + if err != nil { + return nil, err + } + + var buf bytes.Buffer + gzipWriter := gzip.NewWriter(&buf) + _, err = io.Copy(gzipWriter, tw.Buffer) + if err != nil { + return nil, err + } + if err := gzipWriter.Close(); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func sendTarToAPI(tarData []byte, fileName string) error { + mctlCfg, err := config.GetMesheryCtl(viper.GetViper()) + if err != nil { + utils.Log.Error(err) + return err + } + + baseURL := mctlCfg.GetBaseMesheryURL() + url := baseURL + "/api/meshmodels/registers" + + var b bytes.Buffer + writer := multipart.NewWriter(&b) + + formFile, err := writer.CreateFormFile("file", filepath.Base(fileName)) + if err != nil { + utils.Log.Error(fmt.Errorf("failed to create form file: %w", err)) + return err + } + + _, err = formFile.Write(tarData) + if err != nil { + utils.Log.Error(fmt.Errorf("failed to write tar data to form file: %w", err)) + return err + } + + err = writer.Close() + if err != nil { + utils.Log.Error(fmt.Errorf("failed to close writer: %w", err)) + return err + } + + req, err := utils.NewRequest(http.MethodPost, url, &b) + if err != nil { + utils.Log.Error(fmt.Errorf("failed to create request: %w", err)) + return err + } + + req.Header.Set("Content-Type", writer.FormDataContentType()) + resp, err := utils.MakeRequest(req) + if err != nil { + utils.Log.Error(fmt.Errorf("failed to send request: %w", err)) + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + err = models.ErrDoRequest(err, resp.Request.Method, url) + utils.Log.Error(err) + return err + } + utils.Log.Info("Models imported successfully") + return nil +} + +func init() { + importModelCmd.Flags().SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName { + return pflag.NormalizedName(strings.ToLower(name)) + }) + importModelCmd.Flags().StringP("file", "f", "", "Filepath to the directory containing models") +} diff --git a/mesheryctl/internal/cli/root/model/model.go b/mesheryctl/internal/cli/root/model/model.go index 391827804e0..0db2659c819 100644 --- a/mesheryctl/internal/cli/root/model/model.go +++ b/mesheryctl/internal/cli/root/model/model.go @@ -45,7 +45,7 @@ var ( // Color for the whiteboard printer whiteBoardPrinter = color.New(color.FgHiBlack, color.BgWhite, color.Bold) - availableSubcommands = []*cobra.Command{listModelCmd, viewModelCmd, searchModelCmd} + availableSubcommands = []*cobra.Command{listModelCmd, viewModelCmd, searchModelCmd, importModelCmd} countFlag bool ) diff --git a/server/handlers/component_handler.go b/server/handlers/component_handler.go index 9051ece5e58..f76b5c90e29 100644 --- a/server/handlers/component_handler.go +++ b/server/handlers/component_handler.go @@ -3,7 +3,10 @@ package handlers import ( "encoding/json" "fmt" + "io" "net/http" + "os" + "path/filepath" "strconv" "github.com/gofrs/uuid" @@ -13,9 +16,13 @@ import ( "github.com/layer5io/meshery/server/models" "github.com/layer5io/meshery/server/models/pattern/core" "github.com/layer5io/meshkit/models/events" + meshkitutils "github.com/layer5io/meshkit/utils" + + "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/models/meshmodel/registry" + regv1beta1 "github.com/layer5io/meshkit/models/meshmodel/registry/v1beta1" ) @@ -1367,3 +1374,134 @@ func prettifyCompDefSchema(entities []entity.Entity) []v1beta1.ComponentDefiniti } return comps } +func (h *Handler) RegisterMeshmodels(rw http.ResponseWriter, r *http.Request, _ *models.Preference, user *models.User, provider models.Provider) { + file, _, err := r.FormFile("file") + if err != nil { + http.Error(rw, "Error retrieving file", http.StatusBadRequest) + return + } + defer file.Close() + + tempFile, err := os.CreateTemp("", "upload-*.tar.gz") + if err != nil { + http.Error(rw, "Error creating temporary file", http.StatusInternalServerError) + return + } + defer os.Remove(tempFile.Name()) + + _, err = io.Copy(tempFile, file) + if err != nil { + http.Error(rw, "Error saving file", http.StatusInternalServerError) + return + } + + tempDir, err := os.MkdirTemp("", "extracted-") + if err != nil { + http.Error(rw, "Error creating temporary directory", http.StatusInternalServerError) + return + } + defer os.RemoveAll(tempDir) + + err = extractFile(tempFile.Name(), tempDir) + if err != nil { + http.Error(rw, "Error extracting file", http.StatusInternalServerError) + return + } + + err = filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + if meshkitutils.IsYaml(path) { + return processFile(path, h) + } + if meshkitutils.IsTarGz(path) || meshkitutils.IsZip(path) { + return extractFile(path, filepath.Dir(path)) + } + } + return nil + }) + + if err != nil { + http.Error(rw, "Error processing files", http.StatusInternalServerError) + return + } + rw.WriteHeader(http.StatusOK) + rw.Write([]byte("File uploaded and extracted successfully")) +} + +func extractFile(filePath string, destDir string) error { + if meshkitutils.IsTarGz(filePath) { + return meshkitutils.ExtractTarGz(destDir, filePath) + } else if meshkitutils.IsZip(filePath) { + return meshkitutils.ExtractZip(destDir, filePath) + } + return fmt.Errorf("unsupported file type for extraction: %s", filePath) +} + +func processFile(filePath string, h *Handler) error { + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("error opening file: %v", err) + } + defer file.Close() + + content, err := io.ReadAll(file) + if err != nil { + return err + } + + if meshkitutils.IsYaml(filePath) { + return processJSON(content, h) + } + + return fmt.Errorf("unsupported file type: %s", filePath) +} + +func processJSON(content []byte, h *Handler) error { + var tempMap map[string]interface{} + if err := json.Unmarshal(content, &tempMap); err != nil { + return fmt.Errorf("error unmarshalling JSON content: %w", err) + } + + if schemaVersion, ok := tempMap["schemaVersion"].(string); ok { + switch schemaVersion { + case "relationships.meshery.io/v1alpha2": + var rel v1alpha2.RelationshipDefinition + if err := json.Unmarshal(content, &rel); err != nil { + return fmt.Errorf("error unmarshalling JSON content to RelationshipDefinition: %w", err) + } + _, _, err := h.registryManager.RegisterEntity(v1beta1.Host{ + Hostname: rel.Model.Registrant.Hostname, + }, &rel) + if err != nil { + return err + } + h.log.Info("Relationship registered successfully") + return nil + case "core.meshery.io/v1beta1": + if components, ok := tempMap["components"]; ok && components == nil { + return nil + } + if _, ok := tempMap["component"].(map[string]interface{})["kind"].(string); ok { + var comp v1beta1.ComponentDefinition + if err := json.Unmarshal(content, &comp); err != nil { + return fmt.Errorf("error unmarshalling JSON content to ComponentDefinition: %w", err) + } + if comp.Model.Registrant.Hostname != "" { + isRegistrantError, isModelError, err := h.registryManager.RegisterEntity(v1beta1.Host{ + Hostname: comp.Model.Registrant.Hostname, + }, &comp) + if err != nil { + return fmt.Errorf("isRegistrantError: %v, isModelError: %v, error: %w", isRegistrantError, isModelError, err) + } + h.log.Info("Component registered successfully") + return nil + } + } + } + } + + return fmt.Errorf("error unmarshalling JSON content: unknown type") +} diff --git a/server/models/handlers.go b/server/models/handlers.go index a0fa3850296..af2e213683a 100644 --- a/server/models/handlers.go +++ b/server/models/handlers.go @@ -123,7 +123,7 @@ type HandlerInterface interface { UpdateEntityStatus(rw http.ResponseWriter, r *http.Request, prefObj *Preference, user *User, provider Provider) GetMeshmodelRegistrants(rw http.ResponseWriter, r *http.Request) - + RegisterMeshmodels(w http.ResponseWriter, req *http.Request, prefObj *Preference, user *User, provider Provider) HandleResourceSchemas(rw http.ResponseWriter, r *http.Request) GetMeshmodelComponentByModel(rw http.ResponseWriter, r *http.Request) diff --git a/server/router/server.go b/server/router/server.go index 1f229eb106a..0a8e9a981be 100644 --- a/server/router/server.go +++ b/server/router/server.go @@ -191,6 +191,8 @@ func NewRouter(_ context.Context, h models.HandlerInterface, port int, g http.Ha gMux.Handle("/api/meshmodels/registrants", h.ProviderMiddleware(h.AuthMiddleware(http.HandlerFunc(h.GetMeshmodelRegistrants), models.NoAuth))).Methods("GET") + gMux.Handle("/api/meshmodels/registers", h.ProviderMiddleware(h.AuthMiddleware(h.SessionInjectorMiddleware(h.RegisterMeshmodels), models.ProviderAuth))). + Methods("POST") gMux.Handle("/api/meshmodels/categories/{category}", h.ProviderMiddleware(h.AuthMiddleware(http.HandlerFunc(h.GetMeshmodelCategoriesByName), models.NoAuth))).Methods("GET") gMux.Handle("/api/meshmodels/categories/{category}/models", h.ProviderMiddleware(h.AuthMiddleware(http.HandlerFunc(h.GetMeshmodelModelsByCategories), models.NoAuth))).Methods("GET") gMux.Handle("/api/meshmodels/categories/{category}/models/{model}", h.ProviderMiddleware(h.AuthMiddleware(http.HandlerFunc(h.GetMeshmodelModelsByCategoriesByModel), models.NoAuth))).Methods("GET") From 8d0db659e9e70875301f2d18693b50dd564aa246 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Fri, 5 Jul 2024 19:39:36 +0530 Subject: [PATCH 2/7] support for file and oci image w/signoff Signed-off-by: Jougan-0 --- mesheryctl/internal/cli/root/model/import.go | 93 +++++++++++---- server/handlers/component_handler.go | 117 ++++++++++++------- 2 files changed, 150 insertions(+), 60 deletions(-) diff --git a/mesheryctl/internal/cli/root/model/import.go b/mesheryctl/internal/cli/root/model/import.go index 13628720587..b1e210dc108 100644 --- a/mesheryctl/internal/cli/root/model/import.go +++ b/mesheryctl/internal/cli/root/model/import.go @@ -15,7 +15,6 @@ import ( "github.com/layer5io/meshery/mesheryctl/pkg/utils" "github.com/layer5io/meshery/server/models" meshkitutils "github.com/layer5io/meshkit/utils" - "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -24,9 +23,11 @@ import ( var importModelCmd = &cobra.Command{ Use: "import", Short: "import models from mesheryctl command", - Long: "import model by specifying the directory. Use 'import model --file [filepath]' to import models and register them to Meshery.", + Long: "import model by specifying the directory, file, or OCI image. Use 'import model --file [filepath]' or 'import model --dir [directory]' or 'import model --oci [image]' to import models and register them to Meshery.", Example: ` - import model --file /path/to/models + import model --file /path/to/[file.yaml|file.json] + import model --dir /path/to/models + import model --oci docker://example.com/repo:tag `, PreRunE: func(cmd *cobra.Command, args []string) error { // Check prerequisites @@ -49,25 +50,66 @@ var importModelCmd = &cobra.Command{ return nil }, RunE: func(cmd *cobra.Command, args []string) error { - dirpath, err := cmd.Flags().GetString("file") + filePath, err := cmd.Flags().GetString("file") if err != nil { return err } - - if dirpath == "" { - return fmt.Errorf("file path is required") - } - - tarData, err := compressDirectory(dirpath) + dirPath, err := cmd.Flags().GetString("dir") if err != nil { return err } - - fileName := filepath.Base(dirpath) + ".tar.gz" - err = sendTarToAPI(tarData, fileName) + ociImage, err := cmd.Flags().GetString("oci") if err != nil { return err } + fmt.Println(filePath) + if filePath == "" && dirPath == "" && ociImage == "" { + return fmt.Errorf("either file path, directory, or OCI image is required") + } + + if filePath != "" { + if meshkitutils.IsYaml(filePath) { + fileData, err := os.ReadFile(filePath) + if err != nil { + return err + } + fmt.Println("isyaml") + err = sendToAPI(fileData, filePath, "file") + if err != nil { + return err + } + } else if meshkitutils.IsTarGz(filePath) || meshkitutils.IsZip(filePath) { + fileData, err := os.ReadFile(filePath) + if err != nil { + return err + } + err = sendToAPI(fileData, filePath, "dir") + if err != nil { + return err + } + } else { + return fmt.Errorf("unsupported file type: %s", filePath) + } + } + + if dirPath != "" { + tarData, err := compressDirectory(dirPath) + if err != nil { + return err + } + fileName := filepath.Base(dirPath) + ".tar.gz" + err = sendToAPI(tarData, fileName, "dir") + if err != nil { + return err + } + } + + if ociImage != "" { + err := sendToAPI([]byte(ociImage), ociImage, "oci") + if err != nil { + return err + } + } return nil }, @@ -126,7 +168,7 @@ func compressDirectory(dirpath string) ([]byte, error) { return buf.Bytes(), nil } -func sendTarToAPI(tarData []byte, fileName string) error { +func sendToAPI(data []byte, name string, dataType string) error { mctlCfg, err := config.GetMesheryCtl(viper.GetViper()) if err != nil { utils.Log.Error(err) @@ -135,19 +177,27 @@ func sendTarToAPI(tarData []byte, fileName string) error { baseURL := mctlCfg.GetBaseMesheryURL() url := baseURL + "/api/meshmodels/registers" - + fmt.Println(url) var b bytes.Buffer writer := multipart.NewWriter(&b) - formFile, err := writer.CreateFormFile("file", filepath.Base(fileName)) + var formFile io.Writer + if dataType == "file" { + formFile, err = writer.CreateFormFile("file", filepath.Base(name)) + } else if dataType == "oci" { + formFile, err = writer.CreateFormField("oci") + } else { + formFile, err = writer.CreateFormField("dir") + } + if err != nil { - utils.Log.Error(fmt.Errorf("failed to create form file: %w", err)) + utils.Log.Error(fmt.Errorf("failed to create form field: %w", err)) return err } - _, err = formFile.Write(tarData) + _, err = formFile.Write(data) if err != nil { - utils.Log.Error(fmt.Errorf("failed to write tar data to form file: %w", err)) + utils.Log.Error(fmt.Errorf("failed to write data to form field: %w", err)) return err } @@ -162,7 +212,6 @@ func sendTarToAPI(tarData []byte, fileName string) error { utils.Log.Error(fmt.Errorf("failed to create request: %w", err)) return err } - req.Header.Set("Content-Type", writer.FormDataContentType()) resp, err := utils.MakeRequest(req) if err != nil { @@ -184,5 +233,7 @@ func init() { importModelCmd.Flags().SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName { return pflag.NormalizedName(strings.ToLower(name)) }) - importModelCmd.Flags().StringP("file", "f", "", "Filepath to the directory containing models") + importModelCmd.Flags().StringP("file", "f", "", "Filepath to the tar.gz file containing models") + importModelCmd.Flags().StringP("dir", "d", "", "Directory containing models to be tar.gz") + importModelCmd.Flags().StringP("oci", "o", "", "OCI image containing models") } diff --git a/server/handlers/component_handler.go b/server/handlers/component_handler.go index f76b5c90e29..bd29aa91e1b 100644 --- a/server/handlers/component_handler.go +++ b/server/handlers/component_handler.go @@ -1375,6 +1375,31 @@ func prettifyCompDefSchema(entities []entity.Entity) []v1beta1.ComponentDefiniti return comps } func (h *Handler) RegisterMeshmodels(rw http.ResponseWriter, r *http.Request, _ *models.Preference, user *models.User, provider models.Provider) { + ociImage := r.FormValue("oci") + if ociImage != "" { + err := processOCIImage([]byte(ociImage), h) + if err != nil { + http.Error(rw, fmt.Sprintf("Error processing OCI image: %v", err), http.StatusInternalServerError) + return + } + rw.WriteHeader(http.StatusOK) + rw.Write([]byte("OCI image processed successfully")) + return + } + + dirPath := r.FormValue("dir") + if dirPath != "" { + err := processDirectory([]byte(dirPath), h) + if err != nil { + http.Error(rw, fmt.Sprintf("Error processing directory: %v", err), http.StatusInternalServerError) + return + } + + rw.WriteHeader(http.StatusOK) + rw.Write([]byte("Directory processed successfully")) + return + } + file, _, err := r.FormFile("file") if err != nil { http.Error(rw, "Error retrieving file", http.StatusBadRequest) @@ -1382,53 +1407,69 @@ func (h *Handler) RegisterMeshmodels(rw http.ResponseWriter, r *http.Request, _ } defer file.Close() - tempFile, err := os.CreateTemp("", "upload-*.tar.gz") + fileContent, err := io.ReadAll(file) if err != nil { - http.Error(rw, "Error creating temporary file", http.StatusInternalServerError) + http.Error(rw, "Error reading file", http.StatusInternalServerError) return } - defer os.Remove(tempFile.Name()) - _, err = io.Copy(tempFile, file) + err = processFileContent(fileContent, h) if err != nil { - http.Error(rw, "Error saving file", http.StatusInternalServerError) + http.Error(rw, fmt.Sprintf("Error processing file: %v", err), http.StatusInternalServerError) return } + rw.WriteHeader(http.StatusOK) + rw.Write([]byte("File uploaded and processed successfully")) +} + +func processDirectory(data []byte, h *Handler) error { + tempFile, err := os.CreateTemp("", "upload-*.tar.gz") + if err != nil { + return fmt.Errorf("Error creating temporary file: %v", err) + } + defer os.Remove(tempFile.Name()) + + _, err = tempFile.Write(data) + if err != nil { + return fmt.Errorf("Error writing to temporary file: %v", err) + } + + return processUploadedFile(tempFile.Name(), h) +} + +func processUploadedFile(filePath string, h *Handler) error { tempDir, err := os.MkdirTemp("", "extracted-") if err != nil { - http.Error(rw, "Error creating temporary directory", http.StatusInternalServerError) - return + return fmt.Errorf("Error creating temporary directory: %v", err) } defer os.RemoveAll(tempDir) - err = extractFile(tempFile.Name(), tempDir) + err = extractFile(filePath, tempDir) if err != nil { - http.Error(rw, "Error extracting file", http.StatusInternalServerError) - return + return fmt.Errorf("Error extracting file: %v", err) } - err = filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() { if meshkitutils.IsYaml(path) { - return processFile(path, h) + fmt.Println(path) + content, err := os.ReadFile(path) + if err != nil { + return err + } + return processFileContent(content, h) } if meshkitutils.IsTarGz(path) || meshkitutils.IsZip(path) { - return extractFile(path, filepath.Dir(path)) + return processUploadedFile(path, h) } } return nil }) - if err != nil { - http.Error(rw, "Error processing files", http.StatusInternalServerError) - return - } - rw.WriteHeader(http.StatusOK) - rw.Write([]byte("File uploaded and extracted successfully")) + return err } func extractFile(filePath string, destDir string) error { @@ -1440,26 +1481,7 @@ func extractFile(filePath string, destDir string) error { return fmt.Errorf("unsupported file type for extraction: %s", filePath) } -func processFile(filePath string, h *Handler) error { - file, err := os.Open(filePath) - if err != nil { - return fmt.Errorf("error opening file: %v", err) - } - defer file.Close() - - content, err := io.ReadAll(file) - if err != nil { - return err - } - - if meshkitutils.IsYaml(filePath) { - return processJSON(content, h) - } - - return fmt.Errorf("unsupported file type: %s", filePath) -} - -func processJSON(content []byte, h *Handler) error { +func processFileContent(content []byte, h *Handler) error { var tempMap map[string]interface{} if err := json.Unmarshal(content, &tempMap); err != nil { return fmt.Errorf("error unmarshalling JSON content: %w", err) @@ -1484,11 +1506,13 @@ func processJSON(content []byte, h *Handler) error { if components, ok := tempMap["components"]; ok && components == nil { return nil } + if _, ok := tempMap["component"].(map[string]interface{})["kind"].(string); ok { var comp v1beta1.ComponentDefinition if err := json.Unmarshal(content, &comp); err != nil { return fmt.Errorf("error unmarshalling JSON content to ComponentDefinition: %w", err) } + utils.WriteSVGsOnFileSystem(&comp) if comp.Model.Registrant.Hostname != "" { isRegistrantError, isModelError, err := h.registryManager.RegisterEntity(v1beta1.Host{ Hostname: comp.Model.Registrant.Hostname, @@ -1505,3 +1529,18 @@ func processJSON(content []byte, h *Handler) error { return fmt.Errorf("error unmarshalling JSON content: unknown type") } + +func processOCIImage(data []byte, h *Handler) error { + tempFile, err := os.CreateTemp("", "upload-*.oci.tar.gz") + if err != nil { + return fmt.Errorf("Error creating temporary file: %v", err) + } + defer os.Remove(tempFile.Name()) + + _, err = tempFile.Write(data) + if err != nil { + return fmt.Errorf("Error writing to temporary file: %v", err) + } + + return processUploadedFile(tempFile.Name(), h) +} From 3b1da754395c60dfadd1b454586b8078b71addc4 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Fri, 5 Jul 2024 19:43:02 +0530 Subject: [PATCH 3/7] lint fix w/signoff Signed-off-by: Jougan-0 --- mesheryctl/internal/cli/root/model/import.go | 1 - server/handlers/component_handler.go | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mesheryctl/internal/cli/root/model/import.go b/mesheryctl/internal/cli/root/model/import.go index b1e210dc108..5f1b687af31 100644 --- a/mesheryctl/internal/cli/root/model/import.go +++ b/mesheryctl/internal/cli/root/model/import.go @@ -62,7 +62,6 @@ var importModelCmd = &cobra.Command{ if err != nil { return err } - fmt.Println(filePath) if filePath == "" && dirPath == "" && ociImage == "" { return fmt.Errorf("either file path, directory, or OCI image is required") } diff --git a/server/handlers/component_handler.go b/server/handlers/component_handler.go index bd29aa91e1b..562baa23f55 100644 --- a/server/handlers/component_handler.go +++ b/server/handlers/component_handler.go @@ -1383,7 +1383,7 @@ func (h *Handler) RegisterMeshmodels(rw http.ResponseWriter, r *http.Request, _ return } rw.WriteHeader(http.StatusOK) - rw.Write([]byte("OCI image processed successfully")) + _, _ = rw.Write([]byte("OCI image processed successfully")) return } @@ -1396,7 +1396,7 @@ func (h *Handler) RegisterMeshmodels(rw http.ResponseWriter, r *http.Request, _ } rw.WriteHeader(http.StatusOK) - rw.Write([]byte("Directory processed successfully")) + _, _ = rw.Write([]byte("Directory processed successfully")) return } @@ -1420,7 +1420,7 @@ func (h *Handler) RegisterMeshmodels(rw http.ResponseWriter, r *http.Request, _ } rw.WriteHeader(http.StatusOK) - rw.Write([]byte("File uploaded and processed successfully")) + _, _ = rw.Write([]byte("File uploaded and processed successfully")) } func processDirectory(data []byte, h *Handler) error { From 0a232a1fe704716ec226a91decb2b62f16b2e085 Mon Sep 17 00:00:00 2001 From: surajgjadhav Date: Tue, 9 Jul 2024 18:51:38 +0530 Subject: [PATCH 4/7] docs: added NPM_TOKEN details under secrets section Signed-off-by: surajgjadhav --- docs/pages/project/contributing/ci-build-and-release.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/pages/project/contributing/ci-build-and-release.md b/docs/pages/project/contributing/ci-build-and-release.md index eb06e172025..cb0695d71ff 100644 --- a/docs/pages/project/contributing/ci-build-and-release.md +++ b/docs/pages/project/contributing/ci-build-and-release.md @@ -41,6 +41,7 @@ Some portions of the workflow require secrets to accomplish their tasks. These s - `SLACK_BOT_TOKEN`: Used for notification of new GitHub stars given to the Meshery repo. - `CYPRESS_RECORD_KEY`: Used for integration with the Layer5 account on Cypress. - `GLOBAL_TOKEN`: Used for securely transmitting performance test results for the None Provider. +- `NPM_TOKEN`: npm authentication token, used to perform authentication against the npm registry in meshery deployment workflow. The Docker Hub user, `mesheryci`, belongs to the "ciusers" team in Docker Hub and acts as the service account under which these automated builds are being pushed. Every time a new Docker Hub repository is created we have to grant “Admin” (in order to update the README in the Docker Hub repository) permissions to the ciusers team. From aa146f38440850b1d021872179ca9666c2c09617 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Tue, 9 Jul 2024 18:59:09 +0530 Subject: [PATCH 5/7] model Import command w/signoff Signed-off-by: Jougan-0 --- go.mod | 2 +- mesheryctl/internal/cli/root/model/import.go | 150 ++++++------ server/handlers/component_handler.go | 234 ++++++++++--------- server/helpers/utils/utils.go | 8 + server/meshmodel/helper.go | 8 + server/router/server.go | 2 +- 6 files changed, 207 insertions(+), 197 deletions(-) diff --git a/go.mod b/go.mod index 850bfad18cb..f2690c449f0 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/jinzhu/copier v0.4.0 github.com/layer5io/gowrk2 v0.6.1 github.com/layer5io/meshery-operator v0.7.0 - github.com/layer5io/meshkit v0.7.46 + github.com/layer5io/meshkit v0.7.47 github.com/layer5io/meshsync v0.6.24 github.com/layer5io/nighthawk-go v1.0.3 github.com/layer5io/service-mesh-performance v0.6.1 diff --git a/mesheryctl/internal/cli/root/model/import.go b/mesheryctl/internal/cli/root/model/import.go index 5f1b687af31..8a7d8d1531e 100644 --- a/mesheryctl/internal/cli/root/model/import.go +++ b/mesheryctl/internal/cli/root/model/import.go @@ -3,31 +3,30 @@ package model import ( "bytes" "compress/gzip" + "errors" "fmt" "io" "mime/multipart" "net/http" "os" "path/filepath" - "strings" "github.com/layer5io/meshery/mesheryctl/internal/cli/root/config" "github.com/layer5io/meshery/mesheryctl/pkg/utils" + "github.com/layer5io/meshery/server/handlers" "github.com/layer5io/meshery/server/models" meshkitutils "github.com/layer5io/meshkit/utils" "github.com/spf13/cobra" - "github.com/spf13/pflag" "github.com/spf13/viper" ) var importModelCmd = &cobra.Command{ Use: "import", Short: "import models from mesheryctl command", - Long: "import model by specifying the directory, file, or OCI image. Use 'import model --file [filepath]' or 'import model --dir [directory]' or 'import model --oci [image]' to import models and register them to Meshery.", + Long: "import model by specifying the directory, file. Use 'import model [filepath]' or 'import model [directory]'.", Example: ` - import model --file /path/to/[file.yaml|file.json] - import model --dir /path/to/models - import model --oci docker://example.com/repo:tag + import model /path/to/[file.yaml|file.json] + import model /path/to/models `, PreRunE: func(cmd *cobra.Command, args []string) error { // Check prerequisites @@ -49,67 +48,63 @@ var importModelCmd = &cobra.Command{ } return nil }, - RunE: func(cmd *cobra.Command, args []string) error { - filePath, err := cmd.Flags().GetString("file") - if err != nil { - return err + Args: func(_ *cobra.Command, args []string) error { + const errMsg = "Usage: mesheryctl model import [ file | filePath ]\nRun 'mesheryctl model import --help' to see detailed help message" + if len(args) == 0 { + return fmt.Errorf("[ file | filepath ] isn't specified\n\n%v", errMsg) + } else if len(args) > 1 { + return fmt.Errorf("too many arguments\n\n%v", errMsg) } - dirPath, err := cmd.Flags().GetString("dir") - if err != nil { - return err - } - ociImage, err := cmd.Flags().GetString("oci") + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + path := args[0] + fileInfo, err := os.Stat(path) if err != nil { + utils.Log.Error(meshkitutils.ErrReadDir(err, path)) return err } - if filePath == "" && dirPath == "" && ociImage == "" { - return fmt.Errorf("either file path, directory, or OCI image is required") - } - - if filePath != "" { - if meshkitutils.IsYaml(filePath) { - fileData, err := os.ReadFile(filePath) + if fileInfo.IsDir() { + tarData, err := compressDirectory(path) + if err != nil { + return err + } + fileName := filepath.Base(path) + ".tar.gz" + err = sendToAPI(tarData, fileName, "dir") + if err != nil { + utils.Log.Error(err) + return err + } + } else { + if meshkitutils.IsYaml(path) { + fileData, err := os.ReadFile(path) if err != nil { - return err + utils.Log.Error(meshkitutils.ErrReadFile(err, path)) + return nil } - fmt.Println("isyaml") - err = sendToAPI(fileData, filePath, "file") + err = sendToAPI(fileData, path, "file") if err != nil { - return err + utils.Log.Error(err) + return nil } - } else if meshkitutils.IsTarGz(filePath) || meshkitutils.IsZip(filePath) { - fileData, err := os.ReadFile(filePath) + } else if meshkitutils.IsTarGz(path) || meshkitutils.IsZip(path) { + fileData, err := os.ReadFile(path) if err != nil { - return err + err = meshkitutils.ErrReadFile(err, path) + utils.Log.Error(err) + return nil } - err = sendToAPI(fileData, filePath, "dir") + err = sendToAPI(fileData, path, "dir") if err != nil { - return err + utils.Log.Error(err) + return nil } } else { - return fmt.Errorf("unsupported file type: %s", filePath) - } - } - - if dirPath != "" { - tarData, err := compressDirectory(dirPath) - if err != nil { - return err - } - fileName := filepath.Base(dirPath) + ".tar.gz" - err = sendToAPI(tarData, fileName, "dir") - if err != nil { - return err + err = utils.ErrInvalidFile(errors.New("invalid file format")) + utils.Log.Error(err) + return nil } } - - if ociImage != "" { - err := sendToAPI([]byte(ociImage), ociImage, "oci") - if err != nil { - return err - } - } - return nil }, } @@ -120,7 +115,7 @@ func compressDirectory(dirpath string) ([]byte, error) { err := filepath.Walk(dirpath, func(path string, info os.FileInfo, err error) error { if err != nil { - return err + return meshkitutils.ErrFileWalkDir(err, path) } if info.IsDir() { @@ -129,18 +124,18 @@ func compressDirectory(dirpath string) ([]byte, error) { file, err := os.Open(path) if err != nil { - return err + return handlers.ErrOpenFile(path) } defer file.Close() fileData, err := io.ReadAll(file) if err != nil { - return err + return meshkitutils.ErrReadFile(err, path) } relPath, err := filepath.Rel(filepath.Dir(dirpath), path) if err != nil { - return err + return meshkitutils.ErrRelPath(err, path) } if err := tw.Compress(relPath, fileData); err != nil { @@ -158,10 +153,10 @@ func compressDirectory(dirpath string) ([]byte, error) { gzipWriter := gzip.NewWriter(&buf) _, err = io.Copy(gzipWriter, tw.Buffer) if err != nil { - return nil, err + return nil, meshkitutils.ErrCopyFile(err) } if err := gzipWriter.Close(); err != nil { - return nil, err + return nil, meshkitutils.ErrCloseFile(err) } return buf.Bytes(), nil @@ -170,69 +165,54 @@ func compressDirectory(dirpath string) ([]byte, error) { func sendToAPI(data []byte, name string, dataType string) error { mctlCfg, err := config.GetMesheryCtl(viper.GetViper()) if err != nil { - utils.Log.Error(err) return err } baseURL := mctlCfg.GetBaseMesheryURL() - url := baseURL + "/api/meshmodels/registers" - fmt.Println(url) + url := baseURL + "/api/meshmodels/register" var b bytes.Buffer writer := multipart.NewWriter(&b) var formFile io.Writer if dataType == "file" { formFile, err = writer.CreateFormFile("file", filepath.Base(name)) - } else if dataType == "oci" { - formFile, err = writer.CreateFormField("oci") } else { formFile, err = writer.CreateFormField("dir") } - if err != nil { - utils.Log.Error(fmt.Errorf("failed to create form field: %w", err)) - return err - } + } _, err = formFile.Write(data) if err != nil { - utils.Log.Error(fmt.Errorf("failed to write data to form field: %w", err)) + err = meshkitutils.ErrWriteFile(err, name) return err } - err = writer.Close() - if err != nil { - utils.Log.Error(fmt.Errorf("failed to close writer: %w", err)) - return err - } + _ = writer.Close() req, err := utils.NewRequest(http.MethodPost, url, &b) if err != nil { - utils.Log.Error(fmt.Errorf("failed to create request: %w", err)) return err } req.Header.Set("Content-Type", writer.FormDataContentType()) resp, err := utils.MakeRequest(req) if err != nil { - utils.Log.Error(fmt.Errorf("failed to send request: %w", err)) return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { err = models.ErrDoRequest(err, resp.Request.Method, url) - utils.Log.Error(err) return err } + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + err = models.ErrDataRead(err, "response body") + return err + } + + bodyString := string(bodyBytes) + utils.Log.Info(bodyString) utils.Log.Info("Models imported successfully") return nil } - -func init() { - importModelCmd.Flags().SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName { - return pflag.NormalizedName(strings.ToLower(name)) - }) - importModelCmd.Flags().StringP("file", "f", "", "Filepath to the tar.gz file containing models") - importModelCmd.Flags().StringP("dir", "d", "", "Directory containing models to be tar.gz") - importModelCmd.Flags().StringP("oci", "o", "", "OCI image containing models") -} diff --git a/server/handlers/component_handler.go b/server/handlers/component_handler.go index 562baa23f55..b2646ba2077 100644 --- a/server/handlers/component_handler.go +++ b/server/handlers/component_handler.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "github.com/gofrs/uuid" "github.com/gorilla/mux" @@ -15,13 +16,13 @@ import ( "github.com/layer5io/meshery/server/helpers/utils" "github.com/layer5io/meshery/server/models" "github.com/layer5io/meshery/server/models/pattern/core" - "github.com/layer5io/meshkit/models/events" - meshkitutils "github.com/layer5io/meshkit/utils" + "github.com/layer5io/meshkit/models/events" "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/models/meshmodel/registry" + meshkitutils "github.com/layer5io/meshkit/utils" regv1beta1 "github.com/layer5io/meshkit/models/meshmodel/registry/v1beta1" ) @@ -1374,173 +1375,186 @@ func prettifyCompDefSchema(entities []entity.Entity) []v1beta1.ComponentDefiniti } return comps } + +// swagger:route POST /api/meshmodel/register RegisterMeshmodels idRegisterMeshmodels +// Handle POST request for registering entites like components and relationships model. +// +// Register model based on thier Schema Version. +// +// responses: +// 200: noContentWrapper + +// request content byte in form value and header of the type in form func (h *Handler) RegisterMeshmodels(rw http.ResponseWriter, r *http.Request, _ *models.Preference, user *models.User, provider models.Provider) { - ociImage := r.FormValue("oci") - if ociImage != "" { - err := processOCIImage([]byte(ociImage), h) + var compCount, relCount int + dirPath := r.FormValue("dir") + if dirPath != "" { + tempFile, err := os.CreateTemp("", "upload-*.tar.gz") if err != nil { - http.Error(rw, fmt.Sprintf("Error processing OCI image: %v", err), http.StatusInternalServerError) + err = meshkitutils.ErrCreateFile(err, "/tmp/upload-*.tar.gz") + h.log.Error(err) + http.Error(rw, err.Error(), http.StatusBadRequest) return } - rw.WriteHeader(http.StatusOK) - _, _ = rw.Write([]byte("OCI image processed successfully")) - return - } + defer os.Remove(tempFile.Name()) - dirPath := r.FormValue("dir") - if dirPath != "" { - err := processDirectory([]byte(dirPath), h) + _, err = tempFile.Write([]byte(dirPath)) if err != nil { - http.Error(rw, fmt.Sprintf("Error processing directory: %v", err), http.StatusInternalServerError) + err = meshkitutils.ErrWriteFile(err, tempFile.Name()) + h.log.Error(err) + http.Error(rw, err.Error(), http.StatusBadRequest) return } + err = processUploadedFile(tempFile.Name(), h, &compCount, &relCount) + if err != nil { + h.log.Error(err) + http.Error(rw, err.Error(), http.StatusBadRequest) + return + } + message := writeMessageString(compCount, relCount) + if message.Len() > 0 { + h.log.Info(message.String()) + } rw.WriteHeader(http.StatusOK) - _, _ = rw.Write([]byte("Directory processed successfully")) + _, _ = rw.Write([]byte(message.String())) return } file, _, err := r.FormFile("file") if err != nil { - http.Error(rw, "Error retrieving file", http.StatusBadRequest) + err = ErrRetrieveData(err) + http.Error(rw, err.Error(), http.StatusBadRequest) return } defer file.Close() fileContent, err := io.ReadAll(file) if err != nil { - http.Error(rw, "Error reading file", http.StatusInternalServerError) + err = meshkitutils.ErrReadFile(err, string(fileContent)) + http.Error(rw, err.Error(), http.StatusInternalServerError) return } - err = processFileContent(fileContent, h) + entityType, err := meshkitutils.FindEntityType(fileContent) if err != nil { - http.Error(rw, fmt.Sprintf("Error processing file: %v", err), http.StatusInternalServerError) + h.log.Error(err) + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + if entityType == "" { + err = meshkitutils.ErrInvalidSchemaVersion + h.log.Error(err) + http.Error(rw, err.Error(), http.StatusInternalServerError) return } + err = RegisterEntity(fileContent, entityType, h) + if err != nil { + h.log.Error(err) + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + message := "" + if entityType == entity.ComponentDefinition { + message = "Successfully registered Component" + } else { + message = "Successfully registered Relationship" + } + h.log.Info(message) rw.WriteHeader(http.StatusOK) - _, _ = rw.Write([]byte("File uploaded and processed successfully")) + _, _ = rw.Write([]byte(message)) } +func writeMessageString(compCount int, relCount int) strings.Builder { -func processDirectory(data []byte, h *Handler) error { - tempFile, err := os.CreateTemp("", "upload-*.tar.gz") - if err != nil { - return fmt.Errorf("Error creating temporary file: %v", err) - } - defer os.Remove(tempFile.Name()) + var message strings.Builder - _, err = tempFile.Write(data) - if err != nil { - return fmt.Errorf("Error writing to temporary file: %v", err) + if compCount > 0 { + message.WriteString(fmt.Sprintf("Total Components Registered: %d", compCount)) } - return processUploadedFile(tempFile.Name(), h) + if relCount > 0 { + if message.Len() > 0 { + message.WriteString(" and ") + } + message.WriteString(fmt.Sprintf("Registered Relationships: %d", relCount)) + } + return message } - -func processUploadedFile(filePath string, h *Handler) error { +func processUploadedFile(filePath string, h *Handler, compCount *int, relCount *int) error { tempDir, err := os.MkdirTemp("", "extracted-") if err != nil { - return fmt.Errorf("Error creating temporary directory: %v", err) + return ErrCreateDir(err, "Error creating temp dir") } defer os.RemoveAll(tempDir) - err = extractFile(filePath, tempDir) + err = utils.ExtractFile(filePath, tempDir) if err != nil { - return fmt.Errorf("Error extracting file: %v", err) + return err } err = filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { if err != nil { - return err + return meshkitutils.ErrFileWalkDir(err, path) } if !info.IsDir() { if meshkitutils.IsYaml(path) { - fmt.Println(path) content, err := os.ReadFile(path) + if err != nil { + return meshkitutils.ErrReadFile(err, path) + } + entityType, err := meshkitutils.FindEntityType(content) if err != nil { return err } - return processFileContent(content, h) + if entityType != "" { + err = RegisterEntity(content, entityType, h) + if err != nil { + return err + } + if entityType == entity.ComponentDefinition { + *compCount++ + } else { + *relCount++ + } + } + } if meshkitutils.IsTarGz(path) || meshkitutils.IsZip(path) { - return processUploadedFile(path, h) + return processUploadedFile(path, h, compCount, relCount) } } return nil }) - return err } - -func extractFile(filePath string, destDir string) error { - if meshkitutils.IsTarGz(filePath) { - return meshkitutils.ExtractTarGz(destDir, filePath) - } else if meshkitutils.IsZip(filePath) { - return meshkitutils.ExtractZip(destDir, filePath) - } - return fmt.Errorf("unsupported file type for extraction: %s", filePath) -} - -func processFileContent(content []byte, h *Handler) error { - var tempMap map[string]interface{} - if err := json.Unmarshal(content, &tempMap); err != nil { - return fmt.Errorf("error unmarshalling JSON content: %w", err) - } - - if schemaVersion, ok := tempMap["schemaVersion"].(string); ok { - switch schemaVersion { - case "relationships.meshery.io/v1alpha2": - var rel v1alpha2.RelationshipDefinition - if err := json.Unmarshal(content, &rel); err != nil { - return fmt.Errorf("error unmarshalling JSON content to RelationshipDefinition: %w", err) - } - _, _, err := h.registryManager.RegisterEntity(v1beta1.Host{ - Hostname: rel.Model.Registrant.Hostname, - }, &rel) - if err != nil { - return err - } - h.log.Info("Relationship registered successfully") - return nil - case "core.meshery.io/v1beta1": - if components, ok := tempMap["components"]; ok && components == nil { - return nil - } - - if _, ok := tempMap["component"].(map[string]interface{})["kind"].(string); ok { - var comp v1beta1.ComponentDefinition - if err := json.Unmarshal(content, &comp); err != nil { - return fmt.Errorf("error unmarshalling JSON content to ComponentDefinition: %w", err) - } - utils.WriteSVGsOnFileSystem(&comp) - if comp.Model.Registrant.Hostname != "" { - isRegistrantError, isModelError, err := h.registryManager.RegisterEntity(v1beta1.Host{ - Hostname: comp.Model.Registrant.Hostname, - }, &comp) - if err != nil { - return fmt.Errorf("isRegistrantError: %v, isModelError: %v, error: %w", isRegistrantError, isModelError, err) - } - h.log.Info("Component registered successfully") - return nil - } - } +func RegisterEntity(content []byte, entityType entity.EntityType, h *Handler) error { + switch entityType { + case entity.ComponentDefinition: + var c v1beta1.ComponentDefinition + err := json.Unmarshal(content, &c) + if err != nil { + return meshkitutils.ErrUnmarshal(err) } + isRegistrantError, isModelError, err := h.registryManager.RegisterEntity(v1beta1.Host{ + Hostname: c.Model.Registrant.Hostname, + }, &c) + helpers.HandleError(v1beta1.Host{ + Hostname: c.Model.Registrant.Hostname, + }, &c, err, isModelError, isRegistrantError) + return nil + case entity.RelationshipDefinition: + var r v1alpha2.RelationshipDefinition + err := json.Unmarshal(content, &r) + if err != nil { + return meshkitutils.ErrUnmarshal(err) + } + isRegistrantError, isModelError, err := h.registryManager.RegisterEntity(v1beta1.Host{ + Hostname: r.Model.Registrant.Hostname, + }, &r) + helpers.HandleError(v1beta1.Host{ + Hostname: r.Model.Registrant.Hostname, + }, &r, err, isModelError, isRegistrantError) + return nil } - - return fmt.Errorf("error unmarshalling JSON content: unknown type") -} - -func processOCIImage(data []byte, h *Handler) error { - tempFile, err := os.CreateTemp("", "upload-*.oci.tar.gz") - if err != nil { - return fmt.Errorf("Error creating temporary file: %v", err) - } - defer os.Remove(tempFile.Name()) - - _, err = tempFile.Write(data) - if err != nil { - return fmt.Errorf("Error writing to temporary file: %v", err) - } - - return processUploadedFile(tempFile.Name(), h) + return meshkitutils.ErrInvalidSchemaVersion } diff --git a/server/helpers/utils/utils.go b/server/helpers/utils/utils.go index f5c890c6740..8b053380823 100644 --- a/server/helpers/utils/utils.go +++ b/server/helpers/utils/utils.go @@ -360,3 +360,11 @@ func FormatToTitleCase(s string) string { c := cases.Title(language.English) return c.String(s) } +func ExtractFile(filePath string, destDir string) error { + if utils.IsTarGz(filePath) { + return utils.ExtractTarGz(destDir, filePath) + } else if utils.IsZip(filePath) { + return utils.ExtractZip(destDir, filePath) + } + return utils.ErrExtractType +} diff --git a/server/meshmodel/helper.go b/server/meshmodel/helper.go index 3f6bef53dd1..76be0c591bb 100644 --- a/server/meshmodel/helper.go +++ b/server/meshmodel/helper.go @@ -227,3 +227,11 @@ func (erh *EntityRegistrationHelper) watchComponents(ctx context.Context) { } } } +func extractFile(filePath string, destDir string) error { + if mutils.IsTarGz(filePath) { + return mutils.ExtractTarGz(destDir, filePath) + } else if mutils.IsZip(filePath) { + return mutils.ExtractZip(destDir, filePath) + } + return mutils.ErrExtractType +} diff --git a/server/router/server.go b/server/router/server.go index 0a8e9a981be..e9fdbf4f7d5 100644 --- a/server/router/server.go +++ b/server/router/server.go @@ -191,7 +191,7 @@ func NewRouter(_ context.Context, h models.HandlerInterface, port int, g http.Ha gMux.Handle("/api/meshmodels/registrants", h.ProviderMiddleware(h.AuthMiddleware(http.HandlerFunc(h.GetMeshmodelRegistrants), models.NoAuth))).Methods("GET") - gMux.Handle("/api/meshmodels/registers", h.ProviderMiddleware(h.AuthMiddleware(h.SessionInjectorMiddleware(h.RegisterMeshmodels), models.ProviderAuth))). + gMux.Handle("/api/meshmodels/register", h.ProviderMiddleware(h.AuthMiddleware(h.SessionInjectorMiddleware(h.RegisterMeshmodels), models.ProviderAuth))). Methods("POST") gMux.Handle("/api/meshmodels/categories/{category}", h.ProviderMiddleware(h.AuthMiddleware(http.HandlerFunc(h.GetMeshmodelCategoriesByName), models.NoAuth))).Methods("GET") gMux.Handle("/api/meshmodels/categories/{category}/models", h.ProviderMiddleware(h.AuthMiddleware(http.HandlerFunc(h.GetMeshmodelModelsByCategories), models.NoAuth))).Methods("GET") From 046ffd17a93945cd44fe7e1b17ff6d3d9279cd80 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Tue, 9 Jul 2024 19:09:19 +0530 Subject: [PATCH 6/7] docs update for import command w/singoff Signed-off-by: Jougan-0 --- docs/_data/mesheryctlcommands/cmds.yml | 7 +++++++ mesheryctl/pkg/utils/error.go | 4 ++++ mesheryctl/pkg/utils/helpers.go | 2 ++ 3 files changed, 13 insertions(+) diff --git a/docs/_data/mesheryctlcommands/cmds.yml b/docs/_data/mesheryctlcommands/cmds.yml index d226af165c8..1116aa2468f 100644 --- a/docs/_data/mesheryctlcommands/cmds.yml +++ b/docs/_data/mesheryctlcommands/cmds.yml @@ -903,6 +903,13 @@ model: output-format: name: -o, --output-format description: (optional) format to display in [json|yaml] (default "yaml") + import: + name: import + description: Import models using a file or filepath directly to the meshery registry + usage: mesheryctl model import [ file | filepath ] + examples: + - import model /path/to/[file.yaml|file.json] + - import model /path/to/models components: name: components diff --git a/mesheryctl/pkg/utils/error.go b/mesheryctl/pkg/utils/error.go index cf40a7efcc6..5b5232fdde3 100644 --- a/mesheryctl/pkg/utils/error.go +++ b/mesheryctl/pkg/utils/error.go @@ -142,6 +142,8 @@ func SystemModelSubError(msg string, cmd string) string { return formatError(msg, cmdModelList) case "view": return formatError(msg, cmdModelView) + case "import": + return formatError(msg, cmdModelImport) default: return formatError(msg, cmdModel) } @@ -315,6 +317,8 @@ func formatError(msg string, cmd cmdType) string { return formatUsageDetails(msg, modelUsageURL) case cmdModelList: return formatUsageDetails(msg, modelListURL) + case cmdModelImport: + return formatUsageDetails(msg, modelImportURl) case cmdModelView: return formatUsageDetails(msg, modelViewURL) case cmdRegistry: diff --git a/mesheryctl/pkg/utils/helpers.go b/mesheryctl/pkg/utils/helpers.go index 568fc6d2c70..bfd1b46e101 100644 --- a/mesheryctl/pkg/utils/helpers.go +++ b/mesheryctl/pkg/utils/helpers.go @@ -87,6 +87,7 @@ const ( tokenUsageURL = docsBaseURL + "reference/mesheryctl/system/token" modelUsageURL = docsBaseURL + "reference/mesheryctl/system/model" modelListURL = docsBaseURL + "reference/mesheryctl/system/model/list" + modelImportURl = docsBaseURL + "reference/mesheryctl/system/model/import" modelViewURL = docsBaseURL + "reference/mesheryctl/system/model/view" registryUsageURL = docsBaseURL + "reference/mesheryctl/system/registry" relationshipUsageURL = docsBaseURL + "reference/mesheryctl/relationships" @@ -154,6 +155,7 @@ const ( cmdToken cmdType = "token" cmdModel cmdType = "model" cmdModelList cmdType = "model list" + cmdModelImport cmdType = "model import" cmdModelView cmdType = "model view" cmdRegistryPublish cmdType = "registry publish" cmdRegistry cmdType = "regisry" From 05e9b9b6f30ef61de08d7ab64ed4b6554284fd8c Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Tue, 9 Jul 2024 19:17:23 +0530 Subject: [PATCH 7/7] fix for docs w/singoff Signed-off-by: Jougan-0 --- docs/pages/reference/mesheryctl-commands.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/reference/mesheryctl-commands.md b/docs/pages/reference/mesheryctl-commands.md index 200870b30c6..85d58f34dfb 100644 --- a/docs/pages/reference/mesheryctl-commands.md +++ b/docs/pages/reference/mesheryctl-commands.md @@ -769,7 +769,7 @@ Installation, troubleshooting and debugging of Meshery and its adapters {% assign command12 = site.data.mesheryctlcommands.cmds.model %} - {{ command12.name }} + {{ command12.name }} {{ command12.description }}