From 51b968089dd28bcb8faf886efb182533f8c1d4f3 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Wed, 17 Jul 2024 19:41:07 +0530 Subject: [PATCH 001/112] update error for ErrInvalidSchemaVersion w/singoff Signed-off-by: Jougan-0 --- utils/error.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/error.go b/utils/error.go index 8e9a42cf..bc0bc710 100644 --- a/utils/error.go +++ b/utils/error.go @@ -60,9 +60,9 @@ var ( ErrInvalidSchemaVersionCode, errors.Alert, []string{"Invalid schema version"}, - []string{"The `schemaVersion` key in the JSON file is either empty or has an incorrect value."}, - []string{"The JSON file schema is not of type 'relationship' or 'component'.", "The `schemaVersion` key in the JSON should be either `relationships.meshery.io` or `component.meshery.io`."}, - []string{"Verify that the `schemaVersion` key in the JSON has the correct value."}, + []string{"The `schemaVersion` key is either empty or has an incorrect value."}, + []string{"The JSON file schema is not of type 'relationship', 'component', 'model' , 'policy'."}, + []string{"Verify that `schemaVersion` key should be either `relationships.meshery.io`, `component.meshery.io`, `model.meshery.io` or `policy.meshery.io`."}, ) ) From f080ea4861e68484a96173f57a261dbcffb8bb2e Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Wed, 17 Jul 2024 19:42:45 +0530 Subject: [PATCH 002/112] update error for ErrInvalidSchemaVersion w/singoff Signed-off-by: Jougan-0 --- utils/error.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/error.go b/utils/error.go index bc0bc710..3981a282 100644 --- a/utils/error.go +++ b/utils/error.go @@ -61,7 +61,7 @@ var ( errors.Alert, []string{"Invalid schema version"}, []string{"The `schemaVersion` key is either empty or has an incorrect value."}, - []string{"The JSON file schema is not of type 'relationship', 'component', 'model' , 'policy'."}, + []string{"The schema is not of type 'relationship', 'component', 'model' , 'policy'."}, []string{"Verify that `schemaVersion` key should be either `relationships.meshery.io`, `component.meshery.io`, `model.meshery.io` or `policy.meshery.io`."}, ) ) From 7b20d127e3a38b2d048805de2c0b0b2c6b2371ec Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Mon, 29 Jul 2024 19:04:42 +0530 Subject: [PATCH 003/112] add oci support and refractor errorw/signoff Signed-off-by: Jougan-0 --- models/oci/utils.go | 80 +++++++++++++++++++ schemas/configuration/modelImport.json | 65 +++++++++++++++ .../configuration/uiSchemaModelImport.json | 6 ++ schemas/schemaProvider.go | 2 + 4 files changed, 153 insertions(+) create mode 100644 schemas/configuration/modelImport.json create mode 100644 schemas/configuration/uiSchemaModelImport.json diff --git a/models/oci/utils.go b/models/oci/utils.go index 7235f4ab..99cc1002 100644 --- a/models/oci/utils.go +++ b/models/oci/utils.go @@ -1,10 +1,16 @@ package oci import ( + "bytes" + "compress/gzip" + "encoding/json" "fmt" + "io" "os" "path/filepath" + archiveTar "archive/tar" + "github.com/fluxcd/pkg/tar" "github.com/google/go-containerregistry/pkg/crane" gcrv1 "github.com/google/go-containerregistry/pkg/v1" @@ -63,3 +69,77 @@ func UnCompressOCIArtifact(source, destination string) error { return nil } + +// ValidateOCIArtifact validates the OCI artifact tarball using go-containerregistry's validate function +func ValidateOCIArtifact(tarballPath string) error { + img, err := tarball.ImageFromPath(tarballPath, nil) + if err != nil { + return err + } + + return validate.Image(img) +} + +// IsOCIArtifact checks if the tarball is an OCI artifact by looking for manifest.json or index.json +func IsOCIArtifact(data []byte) bool { + reader := bytes.NewReader(data) + var tr *archiveTar.Reader + + if gzr, err := gzip.NewReader(reader); err == nil { + defer gzr.Close() + tr = archiveTar.NewReader(gzr) + } else { + reader.Seek(0, io.SeekStart) + tr = archiveTar.NewReader(reader) + } + + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return false + } + + if header.Name == "manifest.json" || header.Name == "index.json" { + return true + } + } + + return false +} + +func ExtractAndValidateManifest(data []byte) error { + reader := bytes.NewReader(data) + var tr *archiveTar.Reader + + if gzr, err := gzip.NewReader(reader); err == nil { + defer gzr.Close() + tr = archiveTar.NewReader(gzr) + } else { + reader.Seek(0, io.SeekStart) + tr = archiveTar.NewReader(reader) + } + + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + if header.Name == "manifest.json" { + var manifest []map[string]interface{} + if err := json.NewDecoder(tr).Decode(&manifest); err != nil { + return fmt.Errorf("failed to decode manifest.json: %w", err) + } + fmt.Println("manifest.json is valid:", manifest) + return nil + } + } + + return fmt.Errorf("manifest.json not found in tarball") +} diff --git a/schemas/configuration/modelImport.json b/schemas/configuration/modelImport.json new file mode 100644 index 00000000..e3693d40 --- /dev/null +++ b/schemas/configuration/modelImport.json @@ -0,0 +1,65 @@ +{ + "type": "object", + "properties": { + "uploadType": { + "title": "Upload method", + "enum": [ + "File Upload", + "URL Import" + ], + "default": "Select the Upload Method", + "x-rjsf-grid-area": "12", + "description": "Choose the method you prefer to upload your model file. Select 'File Upload' if you have the file on your local system or 'URL Import' if you have the file hosted online." + } + }, + "allOf": [ + { + "if": { + "properties": { + "uploadType": { + "const": "File Upload" + } + } + }, + "then": { + "properties": { + "file": { + "type": "string", + "format": "file", + "description": "Browse the filter file from your file system", + "x-rjsf-grid-area": "12" + } + }, + "required": [ + "file" + ] + } + }, + { + "if": { + "properties": { + "uploadType": { + "const": "URL Import" + } + } + }, + "then": { + "properties": { + "url": { + "type": "string", + "format": "uri", + "title": "URL", + "description": "Provide the URL of the design file you want to import. This should be a direct URL to the file, for example: https://raw.github.com/your-design-file.yaml", + "x-rjsf-grid-area": "12" + } + }, + "required": [ + "url" + ] + } + } + ], + "required": [ + "uploadType" + ] +} diff --git a/schemas/configuration/uiSchemaModelImport.json b/schemas/configuration/uiSchemaModelImport.json new file mode 100644 index 00000000..038796fd --- /dev/null +++ b/schemas/configuration/uiSchemaModelImport.json @@ -0,0 +1,6 @@ +{ + "uploadType": { + "ui:widget": "radio" + }, + "ui:order" : [ "uploadType", "file", "url"] + } diff --git a/schemas/schemaProvider.go b/schemas/schemaProvider.go index 9cafddef..8e871e68 100644 --- a/schemas/schemaProvider.go +++ b/schemas/schemaProvider.go @@ -13,6 +13,7 @@ func getSchemaMap() map[string]string { "helmRepo": "connections/helmConnection/helmRepoConnection.json", "environment": "configuration/environment.json", "workspace": "configuration/workspace.json", + "model": "configuration/modelImport.json", } } @@ -25,6 +26,7 @@ func getUiSchemaMap() map[string]string { "helmRepo": "connections/helmConnection/uiHelmRepoConnection.json", "environment": "configuration/uiSchemaEnvironment.json", "workspace": "configuration/uiSchemaWorkspace.json", + "model": "configuration/uiSchemaModelImport.json", } } From 25cb55b1025a9a8258e8d5be338651884a241655 Mon Sep 17 00:00:00 2001 From: Nithish Karthik Date: Mon, 5 Aug 2024 20:29:33 +0530 Subject: [PATCH 004/112] feat: Move registration logic from meshery server Signed-off-by: Nithish Karthik --- models/registration/dir.go | 91 +++++++++++++++++ models/registration/error.go | 85 ++++++++++++++++ models/registration/helper.go | 61 ++++++++++++ models/registration/interface.go | 10 ++ models/registration/logs.go | 161 +++++++++++++++++++++++++++++++ models/registration/oci.go | 16 +++ models/registration/register.go | 153 +++++++++++++++++++++++++++++ models/registration/tar.go | 10 ++ models/registration/utils.go | 57 +++++++++++ 9 files changed, 644 insertions(+) create mode 100644 models/registration/dir.go create mode 100644 models/registration/error.go create mode 100644 models/registration/helper.go create mode 100644 models/registration/interface.go create mode 100644 models/registration/logs.go create mode 100644 models/registration/oci.go create mode 100644 models/registration/register.go create mode 100644 models/registration/tar.go create mode 100644 models/registration/utils.go diff --git a/models/registration/dir.go b/models/registration/dir.go new file mode 100644 index 00000000..7e341ed6 --- /dev/null +++ b/models/registration/dir.go @@ -0,0 +1,91 @@ +package registration + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + + "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/utils" +) + +type Dir struct { + dirpath string +} + +/* + The directory should contain one and only one `model`. + A directory containing multiple `model` will be invalid. +*/ +func NewDir(path string) Dir { + return Dir{dirpath: path} +} + + +/* PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the packagingUnit struct. +*/ +func (d Dir) PkgUnit() (_ packagingUnit, err error){ + pkg := packagingUnit{} + // check if the given is a directory + _, err = os.ReadDir(d.dirpath) + if(err != nil){ + return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not read the directory: %e", err)) + } + err = filepath.Walk(d.dirpath, func (path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + if(f.IsDir()){ + return nil + } + byt, _ := os.ReadFile(path) + if(byt == nil){ + return nil + } + + var e entity.Entity + e, err = getEntity(byt) + if err != nil { + RegLog.AddInvalidDefinition(path, err) + return nil + } + // set it to pkgunit + switch e.Type() { + case entity.Model: + if !reflect.ValueOf(pkg.model).IsZero() { + // currently models inside models are not handled + return nil + } + model, err := utils.Cast[*v1beta1.Model](e) + if(err != nil){ + RegLog.AddInvalidDefinition(path, ErrGetEntity(err)) + } + pkg.model = *model + case entity.ComponentDefinition: + comp , err := utils.Cast[*v1beta1.ComponentDefinition](e) + if(err != nil){ + RegLog.AddInvalidDefinition(path, ErrGetEntity(err)) + } + pkg.components = append(pkg.components, *comp) + case entity.RelationshipDefinition: + rel, err := utils.Cast[*v1alpha2.RelationshipDefinition](e) + if(err != nil){ + RegLog.AddInvalidDefinition(path, ErrGetEntity(err)) + } + pkg.relationships = append(pkg.relationships, *rel) + } + return nil + }) + if err != nil { + return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not completely walk the file tree: %e", err)) + } + if (reflect.ValueOf(pkg.model).IsZero()){ + err := fmt.Errorf("Model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core....... ...... must be present, otherwise the model package is considered malformed..") + RegLog.invalidDefinitions[d.dirpath] = err + return pkg, err + } + return pkg, err +} diff --git a/models/registration/error.go b/models/registration/error.go new file mode 100644 index 00000000..73af3795 --- /dev/null +++ b/models/registration/error.go @@ -0,0 +1,85 @@ +package registration + +import ( + "fmt" + + "github.com/layer5io/meshkit/errors" +) + +const ( + ErrDirPkgUnitParseFailCode = "replace_me" + ErrGetEntityCode = "replace_me" + ErrRegisterEntityCode = "replace_me" + ErrImportFailureCode = "replace_me" + ErrMissingRegistrantCode = "replace_me" + ErrSeedingComponentsCode = "replace-me" +) + + + +func ErrSeedingComponents(err error) error { + return errors.New( + ErrSeedingComponentsCode, + errors.Alert, + []string{"Failed to register the given models into meshery's registry"}, + []string{err.Error()}, + []string{"Given models may not be in accordance with Meshery's schema", "Internal(OS level) error while reading files" }, + []string{"Make sure the models being seeded are valid in accordance with Meshery's schema", "If it is an internal error, please try again after some time"}, + ) +} + +func ErrMissingRegistrant(modelName string) error { + return errors.New( + ErrMissingRegistrantCode, + errors.Alert, + []string{fmt.Sprintf("Model with name: %s does not have registrant information", modelName)}, + []string{"Meshery models are always registered in context of a registrant."}, + // there is only one cause for this error + []string{""}, + []string{"Make sure that the registrant information is present in the model definition"}, + ) +} + +func ErrRegisterEntity(err error, name, entity string) error { + return errors.New( + ErrRegisterEntityCode, + errors.Alert, + []string{fmt.Sprintf("Failed to register an entity of name: %s and type: %s into Meshery's registry", name, entity)}, + []string{err.Error()}, + []string{fmt.Sprintf("%s definition might be violating the definition schema", entity), fmt.Sprintf("%s might be missing model details", entity)}, + []string{fmt.Sprintf("ensure the %s definition follows the correct schema", entity), fmt.Sprintf("ensure %s definition belongs to correct model", entity)}, + ) +} + +func ErrGetEntity(err error) error { + return errors.New( + ErrGetEntityCode, + errors.Alert, + []string{"Could not parse the given data into any Meshery entity"}, + []string{err.Error()}, + []string{"Entity definition might not be in accordance with it's corresponding schema", "Might be an invalid/unsupported schemaVersion"}, + []string{"Ensure that the definition given is in accordance with it's schema", "Ensure that the schemaVersion used is valid and is supported by Meshery"}, + ) +} + +func ErrDirPkgUnitParseFail(dirpath string, err error) error { + return errors.New( + ErrDirPkgUnitParseFailCode, + errors.Alert, + []string{fmt.Sprintf("Directory at path: %s cannot be registered into Meshery", dirpath)}, + []string{fmt.Sprint(err.Error())}, + []string{"The directory might not have a valid model definition", "Might be some internal issues while walking the file tree"}, + []string{"Make sure that there is a valid model definition present in the directory. Meshery's registration pipeline currently does not support nested models, therefore the behaviour might be unexpected if it contains nested models.", "If there is an internal error, please try again after some time"}, + ) +} + +func ErrImportFailure(hostname string, failedMsg string) error { + return errors.New( + ErrImportFailureCode, + errors.Alert, + []string{fmt.Sprintf("Errors while registering entities for registrant: %s", hostname)}, + []string{failedMsg}, + []string{"Entity definition might not be in accordance with schema", "Entity version might not be supported by Meshery"}, + []string{"See the registration logs (found at $HOME/.meshery/logs/registry/registry-logs.log) to find out which Entity failed to be imported with more specific error information."}, + ) +} \ No newline at end of file diff --git a/models/registration/helper.go b/models/registration/helper.go new file mode 100644 index 00000000..db3b2d7d --- /dev/null +++ b/models/registration/helper.go @@ -0,0 +1,61 @@ +package registration + +import ( + "os" + "path/filepath" + + "github.com/layer5io/meshery/server/models" + "github.com/layer5io/meshkit/logger" + meshmodel "github.com/layer5io/meshkit/models/meshmodel/registry" +) + +var ModelsPath = "../meshmodel" + +func GetModelDirectoryPaths() ([]string, error) { + dirEntries := make([]string, 0) + modelsDirs, err := os.ReadDir(ModelsPath) + if(err != nil){ + return dirEntries, err + } + for _, modelDir := range modelsDirs { + if(!modelDir.IsDir()){ + continue + } + modelVersionsDirPath := filepath.Join(ModelsPath, modelDir.Name()) + modelVersionsDir, err := os.ReadDir(modelVersionsDirPath) + if(err != nil){ + return dirEntries, err + } + for _, version := range modelVersionsDir { + if(!version.IsDir()){ + continue + } + modelDefVersionsDirPath := filepath.Join(modelVersionsDirPath, version.Name()) + modelDefVersionsDir, err := os.ReadDir(modelDefVersionsDirPath) + if(err != nil){ + return dirEntries, err + } + for _, defVersion := range modelDefVersionsDir { + if(!defVersion.IsDir()){ + continue + } + defPath := filepath.Join(modelDefVersionsDirPath, defVersion.Name()) + dirEntries = append(dirEntries, defPath) + } + } + } + return dirEntries, nil +} + +func SeedComponents(log logger.Handler, hc *models.HandlerConfig, regm *meshmodel.RegistryManager) { + regHelper := NewRegistrationHelper(log, hc, regm) + modelDirPaths, err := GetModelDirectoryPaths() + if(err != nil){ + log.Error(ErrSeedingComponents(err)) + } + for _, dirPath := range modelDirPaths { + dir := NewDir(dirPath) + regHelper.Register(dir) + } + regHelper.RegistryLog() +} \ No newline at end of file diff --git a/models/registration/interface.go b/models/registration/interface.go new file mode 100644 index 00000000..d9655bfe --- /dev/null +++ b/models/registration/interface.go @@ -0,0 +1,10 @@ +package registration + +// Anything that can be parsed into a packagingUnit is a RegisterableEntity in Meshery server +type RegisterableEntity interface { + /* + 1. `err` - this is a breaking error, which signifies that the given entity is invalid + 2. Errors encountered while parsing items into meshmodel entites are stored in the registration logger + */ + PkgUnit() (packagingUnit, error) +} \ No newline at end of file diff --git a/models/registration/logs.go b/models/registration/logs.go new file mode 100644 index 00000000..3632824f --- /dev/null +++ b/models/registration/logs.go @@ -0,0 +1,161 @@ +package registration + +import ( + "fmt" + "strings" + + "github.com/layer5io/meshery/server/models" + "github.com/layer5io/meshkit/models/events" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/spf13/viper" + mutils "github.com/layer5io/meshkit/utils" + meshmodel "github.com/layer5io/meshkit/models/meshmodel/registry" + + gofrs "github.com/gofrs/uuid" +) + +var RegLog *registrationFailureLog +var TAB = " " + +// representation of what could not be registered +type registrationFailureLog struct { + // {'artifacthub':{'modelname': 'component' : {'jaegar': error}}} + // for models, the structure will be like this: + // {'artifacthub': {'': 'model': {'model': error}}} + // think of it like every model belongs to a global model called '' + failureData map[string](map[string](map[string]((map[string]error)))) + + // invalid definitions while parsing the given input (oci, tar, dir) into meshmodel entities + invalidDefinitions map[string](error) +} + +func init() { + RegLog = ®istrationFailureLog{ + failureData: make(map[string](map[string](map[string]((map[string]error))))), + invalidDefinitions: make(map[string]error), + } +} + +func (rfl *registrationFailureLog) AddInvalidDefinition(path string, err error){ + rfl.invalidDefinitions[path] = err +} + +func (rfl *registrationFailureLog) InsertEntityRegFailure(hostname string, modelName string,entityType entity.EntityType, name string, err error){ + if(rfl.failureData[hostname] == nil ) { + rfl.failureData[hostname] = make(map[string]map[string]map[string]error) + } + if(rfl.failureData[hostname][modelName] == nil){ + rfl.failureData[hostname][modelName] = make(map[string]map[string]error) + } + if(rfl.failureData[hostname][modelName][string(entityType)] == nil){ + rfl.failureData[hostname][modelName][string(entityType)] = make(map[string]error) + } + rfl.failureData[hostname][modelName][string(entityType)][name] = err +} + +func (rfl *registrationFailureLog) GetNonRegisteredEntitiesCount()map[string]v1beta1.EntitySummary{ + res := make(map[string]v1beta1.EntitySummary) + for registrant, modelNamespacedData := range rfl.failureData { + entitySummary := v1beta1.EntitySummary{} + for modelName, entityTypeNamespacedData := range modelNamespacedData { + if(modelName == ""){ + modelsCount := len(entityTypeNamespacedData[string(entity.Model)]) + entitySummary.Models = int64(modelsCount) + continue + } + entitySummary.Components = int64(len(entityTypeNamespacedData[string(entity.ComponentDefinition)])) + entitySummary.Relationships = int64(len(entityTypeNamespacedData[string(entity.RelationshipDefinition)])) + entitySummary.Policies = int64(len(entityTypeNamespacedData[string(entity.PolicyDefinition)])) + } + res[registrant] = entitySummary + } + return res +} + + +func failedMsgCompute(failedMsg string, hostName string) (string, error) { + nonImportModel := RegLog.GetNonRegisteredEntitiesCount()[hostName] + + if nonImportModel.Models > 0 || nonImportModel.Components > 0 || nonImportModel.Relationships > 0 || nonImportModel.Policies > 0 { + failedMsg = "Failed to import" + appendIfNonZero := func(msg string, count int64, entityName string) string { + if count > 0 { + return fmt.Sprintf("%s %d %s", msg, count, entityName) + } + return msg + } + + failedMsg = appendIfNonZero(failedMsg, nonImportModel.Models, "models") + failedMsg = appendIfNonZero(failedMsg, nonImportModel.Components, "components") + failedMsg = appendIfNonZero(failedMsg, nonImportModel.Relationships, "relationships") + failedMsg = appendIfNonZero(failedMsg, nonImportModel.Policies, "policies") + } + return failedMsg, nil +} + +func FailedEventCompute(hostname string, mesheryInstanceID gofrs.UUID, provider *models.Provider, userID string, ec *models.Broadcast) (string, error) { + + failedMsg, err := failedMsgCompute("", hostname) + if err != nil { + return "", err + } + if failedMsg != "" { + filePath := viper.GetString("REGISTRY_LOG_FILE") + errorEventBuilder := events.NewEvent().FromUser(mesheryInstanceID).FromSystem(mesheryInstanceID).WithCategory("registration").WithAction("get_summary") + errorEventBuilder.WithSeverity(events.Error).WithDescription(failedMsg) + errorEvent := errorEventBuilder.Build() + errorEventBuilder.WithMetadata(map[string]interface{}{ + "DownloadLink": filePath, + "ViewLink": filePath, + "error": ErrImportFailure(hostname, failedMsg), + }) + _ = (*provider).PersistEvent(errorEvent) + if userID != "" { + userUUID := gofrs.FromStringOrNil(userID) + ec.Publish(userUUID, errorEvent) + + } + } + return failedMsg, nil + +} + + +func writeLogsToFiles() error { + filePath := viper.GetString("REGISTRY_LOG_FILE") + // Initialize the formatted log message + var logMessage strings.Builder + + logMessage.WriteString("Invalid Definitions: \n") + for path, err := range RegLog.invalidDefinitions { + logMessage.WriteString(TAB + "[" + path + "] " + err.Error() + " \n" ) + } + logMessage.WriteString("---------------------------------------- \n") + for host, modelNamespacedData := range RegLog.failureData { + logMessage.WriteString(fmt.Sprintf("%s failed to register:\n", host)) + for modelName, entityTypeNamespacedData := range modelNamespacedData { + if(modelName == "") { + logMessage.WriteString(fmt.Sprintf("%sModels:\n", TAB)) + for name, modelData := range entityTypeNamespacedData[string(entity.Model)]{ + logMessage.WriteString(TAB + TAB + name + ": " + modelData.Error()+ "\n") + } + continue + } + logMessage.WriteString(fmt.Sprintf("%sFor Model %s:\n", TAB, modelName)) + for entityType, entityNameNamespacedData := range entityTypeNamespacedData { + logMessage.WriteString(fmt.Sprintf("%s%s%s:\n", TAB,TAB, entityType)) + for name, err := range entityNameNamespacedData { + logMessage.WriteString(fmt.Sprintf("%s%s%s%s: %s\n", TAB,TAB, TAB, name, err.Error() )) + } + + } + } + } + // Write the formatted log message to the file + err := mutils.WriteToFile(filePath, logMessage.String()) + if err != nil { + return meshmodel.ErrWritingRegisteryAttempts(err) + } + return nil +} \ No newline at end of file diff --git a/models/registration/oci.go b/models/registration/oci.go new file mode 100644 index 00000000..75ff41b2 --- /dev/null +++ b/models/registration/oci.go @@ -0,0 +1,16 @@ +package registration + + +import( + gcrv1 "github.com/google/go-containerregistry/pkg/v1" + +) + +type OCIImage struct { + _ gcrv1.Image +} + +func (o OCIImage) PkgUnit() (packagingUnit, error){ + pkg := packagingUnit{} + return pkg, nil +} \ No newline at end of file diff --git a/models/registration/register.go b/models/registration/register.go new file mode 100644 index 00000000..78ac51d7 --- /dev/null +++ b/models/registration/register.go @@ -0,0 +1,153 @@ +package registration + +import ( + "fmt" + + "github.com/gofrs/uuid" + "github.com/layer5io/meshery/server/models" + "github.com/layer5io/meshkit/logger" + "github.com/layer5io/meshkit/models/events" + "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/entity" + meshmodel "github.com/layer5io/meshkit/models/meshmodel/registry" + "github.com/spf13/viper" +) + +// packaingUnit is the representation of the atomic unit that can be registered into meshery server +type packagingUnit struct { + model v1beta1.Model + components []v1beta1.ComponentDefinition + relationships []v1alpha2.RelationshipDefinition + _ []v1beta1.PolicyDefinition +} + +type RegistrationHelper struct { + handlerConfig *models.HandlerConfig + regManager *meshmodel.RegistryManager + log logger.Handler +} + +func NewRegistrationHelper(log logger.Handler, hc *models.HandlerConfig, regm *meshmodel.RegistryManager) RegistrationHelper { + return RegistrationHelper{handlerConfig: hc, log: log, regManager: regm} +} + +/* + Register will accept a RegisterableEntity (dir, tar or oci for now). + Errors are written to the log file. +*/ +func (rh *RegistrationHelper) Register(entity RegisterableEntity) { + // get the packaging units + pu, err := entity.PkgUnit() + if(err != nil){ + // given input is not a valid model, or could not walk the directory + return + } + // fmt.Printf("Packaging Unit: Model name: %s, comps: %d, rels: %d\n", pu.model.Name, len(pu.components), len(pu.relationships)) + rh.register(pu) +} + + +/* + register will return an error if it is not able to register the `model`. + If there are errors when registering other entities, they are handled properly but does not stop the registration process. +*/ +func (rh *RegistrationHelper)register(pkg packagingUnit) { + // 1. Register the model + model := pkg.model + + // Dont register anything else if registrant is not there + if(model.Registrant.Hostname == ""){ + err := ErrMissingRegistrant(model.Name) + RegLog.InsertEntityRegFailure(model.Registrant.Hostname, "",entity.Model, model.Name, err) + return + } + _, _, err := rh.regManager.RegisterEntity( + v1beta1.Host{Hostname: model.Registrant.Hostname,}, + &model, + ) + + // If model cannot be registered, don't register anything else + if err != nil { + err = ErrRegisterEntity(err, string(model.Type()), model.DisplayName) + RegLog.InsertEntityRegFailure(model.Registrant.Hostname, "",entity.Model, model.Name, err) + return + } + + hostname := model.Registrant.Hostname + modelName := model.Name + // 2. Register components + for _, comp := range pkg.components { + comp.Model = model + _, _, err := rh.regManager.RegisterEntity( + v1beta1.Host{Hostname: hostname,}, + &comp, + ) + if err != nil { + err = ErrRegisterEntity(err, string(comp.Type()), comp.DisplayName) + RegLog.InsertEntityRegFailure(hostname, modelName ,entity.ComponentDefinition, comp.DisplayName, err) + rh.log.Error(err) + } + } + + // 3. Register relationships + for _, rel := range pkg.relationships { + rel.Model = model + _, _, err := rh.regManager.RegisterEntity(v1beta1.Host{ + Hostname: hostname, + }, &rel) + if err != nil { + err = ErrRegisterEntity(err, string(rel.Type()), rel.Kind) + rh.log.Error(err) + RegLog.InsertEntityRegFailure(hostname, modelName ,entity.RelationshipDefinition, rel.ID.String(), err) + } + } +} + +func (erh *RegistrationHelper) RegistryLog() { + log := erh.log + provider := erh.handlerConfig.Providers["None"] + + systemID := viper.GetString("INSTANCE_ID") + + sysID := uuid.FromStringOrNil(systemID) + hosts, _, err := erh.regManager.GetRegistrants(&v1beta1.HostFilter{}) + if err != nil { + log.Error(err) + } + + for _, host := range hosts { + eventBuilder := events.NewEvent().FromSystem(sysID).FromUser(sysID).WithCategory("entity").WithAction("get_summary") + successMessage := fmt.Sprintf("For registrant %s successfully imported", host.Hostname) + appendIfNonZero := func(value int64, label string) { + if value != 0 { + successMessage += fmt.Sprintf(" %d %s", value, label) + } + } + appendIfNonZero(host.Summary.Models, "models") + appendIfNonZero(host.Summary.Components, "components") + appendIfNonZero(host.Summary.Relationships, "relationships") + appendIfNonZero(host.Summary.Policies, "policies") + + log.Info(successMessage) + eventBuilder.WithMetadata(map[string]interface{}{ + "Hostname": host.Hostname, + }) + eventBuilder.WithSeverity(events.Informational).WithDescription(successMessage) + successEvent := eventBuilder.Build() + _ = provider.PersistEvent(successEvent) + + failLog, err := FailedEventCompute(host.Hostname, sysID, &provider, "", erh.handlerConfig.EventBroadcaster) + if err != nil { + log.Error(err) + } + if failLog != "" { + log.Error(meshmodel.ErrRegisteringEntity(failLog, host.Hostname)) + } + + } + err = writeLogsToFiles() + if err != nil { + log.Error(err) + } +} \ No newline at end of file diff --git a/models/registration/tar.go b/models/registration/tar.go new file mode 100644 index 00000000..62625def --- /dev/null +++ b/models/registration/tar.go @@ -0,0 +1,10 @@ +package registration + +type Tar struct { + _ string +} + +func (t Tar) PkgUnit() (packagingUnit, error){ + pkg := packagingUnit{} + return pkg, nil +} \ No newline at end of file diff --git a/models/registration/utils.go b/models/registration/utils.go new file mode 100644 index 00000000..2139d64b --- /dev/null +++ b/models/registration/utils.go @@ -0,0 +1,57 @@ +package registration + +import ( + "encoding/json" + "fmt" + + "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/entity" + "gopkg.in/yaml.v2" +) + +func unmarshal(byt []byte, out interface{}) error { + err := json.Unmarshal(byt, out) + if(err != nil){ + err = yaml.Unmarshal(byt, out) + if(err != nil){ + return fmt.Errorf("Not a valid YAML or JSON") + } + } + return nil +} + +// TODO: refactor this and use CUE +func getEntity(byt []byte) (et entity.Entity, _ error) { + var versionMeta v1beta1.VersionMeta + err := unmarshal(byt, &versionMeta) + if err != nil || versionMeta.SchemaVersion == "" { + return nil, ErrGetEntity(fmt.Errorf("Does not contain versionmeta")) + } + switch (versionMeta.SchemaVersion) { + case v1beta1.ComponentSchemaVersion: + var compDef v1beta1.ComponentDefinition + err := unmarshal(byt, &compDef) + if err != nil { + return nil, ErrGetEntity(fmt.Errorf("Invalid component definition: %s", err.Error())) + } + et = &compDef + case v1beta1.ModelSchemaVersion: + var model v1beta1.Model + err := unmarshal(byt,&model) + if err != nil { + return nil, ErrGetEntity(fmt.Errorf("Invalid model definition: %s", err.Error())) + } + et = &model + case v1alpha2.RelationshipSchemaVersion: + var rel v1alpha2.RelationshipDefinition + err := unmarshal(byt,&rel) + if err != nil { + return nil, ErrGetEntity(fmt.Errorf("Invalid relationship definition: %s", err.Error())) + } + et = &rel + default: + return nil, ErrGetEntity(fmt.Errorf("Not a valid component definition, model definition, or relationship definition")) + } + return et, nil +} \ No newline at end of file From d9b215fd5223af8be1932a22902a5461a158a9f1 Mon Sep 17 00:00:00 2001 From: Nithish Karthik Date: Tue, 6 Aug 2024 11:42:58 +0530 Subject: [PATCH 005/112] refactor: Move log handling to meshery server Signed-off-by: Nithish Karthik --- models/registration/dir.go | 16 +-- models/registration/helper.go | 61 ------------ models/registration/interface.go | 20 +++- models/registration/logs.go | 161 ------------------------------- models/registration/oci.go | 4 +- models/registration/register.go | 90 +++-------------- models/registration/tar.go | 4 +- models/registration/utils.go | 2 +- 8 files changed, 47 insertions(+), 311 deletions(-) delete mode 100644 models/registration/helper.go delete mode 100644 models/registration/logs.go diff --git a/models/registration/dir.go b/models/registration/dir.go index 7e341ed6..2272e8ec 100644 --- a/models/registration/dir.go +++ b/models/registration/dir.go @@ -25,9 +25,11 @@ func NewDir(path string) Dir { } -/* PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the packagingUnit struct. +/* + PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the packagingUnit struct. + Invalid definitions are stored in the regErrStore with error data. */ -func (d Dir) PkgUnit() (_ packagingUnit, err error){ +func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err error){ pkg := packagingUnit{} // check if the given is a directory _, err = os.ReadDir(d.dirpath) @@ -49,7 +51,7 @@ func (d Dir) PkgUnit() (_ packagingUnit, err error){ var e entity.Entity e, err = getEntity(byt) if err != nil { - RegLog.AddInvalidDefinition(path, err) + regErrStore.AddInvalidDefinition(path, err) return nil } // set it to pkgunit @@ -61,19 +63,19 @@ func (d Dir) PkgUnit() (_ packagingUnit, err error){ } model, err := utils.Cast[*v1beta1.Model](e) if(err != nil){ - RegLog.AddInvalidDefinition(path, ErrGetEntity(err)) + regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) } pkg.model = *model case entity.ComponentDefinition: comp , err := utils.Cast[*v1beta1.ComponentDefinition](e) if(err != nil){ - RegLog.AddInvalidDefinition(path, ErrGetEntity(err)) + regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) } pkg.components = append(pkg.components, *comp) case entity.RelationshipDefinition: rel, err := utils.Cast[*v1alpha2.RelationshipDefinition](e) if(err != nil){ - RegLog.AddInvalidDefinition(path, ErrGetEntity(err)) + regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) } pkg.relationships = append(pkg.relationships, *rel) } @@ -84,7 +86,7 @@ func (d Dir) PkgUnit() (_ packagingUnit, err error){ } if (reflect.ValueOf(pkg.model).IsZero()){ err := fmt.Errorf("Model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core....... ...... must be present, otherwise the model package is considered malformed..") - RegLog.invalidDefinitions[d.dirpath] = err + regErrStore.AddInvalidDefinition(d.dirpath, err) return pkg, err } return pkg, err diff --git a/models/registration/helper.go b/models/registration/helper.go deleted file mode 100644 index db3b2d7d..00000000 --- a/models/registration/helper.go +++ /dev/null @@ -1,61 +0,0 @@ -package registration - -import ( - "os" - "path/filepath" - - "github.com/layer5io/meshery/server/models" - "github.com/layer5io/meshkit/logger" - meshmodel "github.com/layer5io/meshkit/models/meshmodel/registry" -) - -var ModelsPath = "../meshmodel" - -func GetModelDirectoryPaths() ([]string, error) { - dirEntries := make([]string, 0) - modelsDirs, err := os.ReadDir(ModelsPath) - if(err != nil){ - return dirEntries, err - } - for _, modelDir := range modelsDirs { - if(!modelDir.IsDir()){ - continue - } - modelVersionsDirPath := filepath.Join(ModelsPath, modelDir.Name()) - modelVersionsDir, err := os.ReadDir(modelVersionsDirPath) - if(err != nil){ - return dirEntries, err - } - for _, version := range modelVersionsDir { - if(!version.IsDir()){ - continue - } - modelDefVersionsDirPath := filepath.Join(modelVersionsDirPath, version.Name()) - modelDefVersionsDir, err := os.ReadDir(modelDefVersionsDirPath) - if(err != nil){ - return dirEntries, err - } - for _, defVersion := range modelDefVersionsDir { - if(!defVersion.IsDir()){ - continue - } - defPath := filepath.Join(modelDefVersionsDirPath, defVersion.Name()) - dirEntries = append(dirEntries, defPath) - } - } - } - return dirEntries, nil -} - -func SeedComponents(log logger.Handler, hc *models.HandlerConfig, regm *meshmodel.RegistryManager) { - regHelper := NewRegistrationHelper(log, hc, regm) - modelDirPaths, err := GetModelDirectoryPaths() - if(err != nil){ - log.Error(ErrSeedingComponents(err)) - } - for _, dirPath := range modelDirPaths { - dir := NewDir(dirPath) - regHelper.Register(dir) - } - regHelper.RegistryLog() -} \ No newline at end of file diff --git a/models/registration/interface.go b/models/registration/interface.go index d9655bfe..a7ee0732 100644 --- a/models/registration/interface.go +++ b/models/registration/interface.go @@ -1,10 +1,24 @@ package registration +import ( + "github.com/layer5io/meshkit/models/meshmodel/entity" +) + +/* + RegistrationErrorStore stores all the errors that does not break the registration process, but have to be reported nevertheless. +*/ +type RegistrationErrorStore interface { + AddInvalidDefinition(string,error) + InsertEntityRegError(hostname string, modelName string, entityType entity.EntityType, entityName string, err error) +} + // Anything that can be parsed into a packagingUnit is a RegisterableEntity in Meshery server type RegisterableEntity interface { /* 1. `err` - this is a breaking error, which signifies that the given entity is invalid - 2. Errors encountered while parsing items into meshmodel entites are stored in the registration logger + 2. Errors encountered while parsing items into meshmodel entites are stored in the RegistrationErrorStore */ - PkgUnit() (packagingUnit, error) -} \ No newline at end of file + PkgUnit(RegistrationErrorStore) (packagingUnit, error) +} + + diff --git a/models/registration/logs.go b/models/registration/logs.go deleted file mode 100644 index 3632824f..00000000 --- a/models/registration/logs.go +++ /dev/null @@ -1,161 +0,0 @@ -package registration - -import ( - "fmt" - "strings" - - "github.com/layer5io/meshery/server/models" - "github.com/layer5io/meshkit/models/events" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" - "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/spf13/viper" - mutils "github.com/layer5io/meshkit/utils" - meshmodel "github.com/layer5io/meshkit/models/meshmodel/registry" - - gofrs "github.com/gofrs/uuid" -) - -var RegLog *registrationFailureLog -var TAB = " " - -// representation of what could not be registered -type registrationFailureLog struct { - // {'artifacthub':{'modelname': 'component' : {'jaegar': error}}} - // for models, the structure will be like this: - // {'artifacthub': {'': 'model': {'model': error}}} - // think of it like every model belongs to a global model called '' - failureData map[string](map[string](map[string]((map[string]error)))) - - // invalid definitions while parsing the given input (oci, tar, dir) into meshmodel entities - invalidDefinitions map[string](error) -} - -func init() { - RegLog = ®istrationFailureLog{ - failureData: make(map[string](map[string](map[string]((map[string]error))))), - invalidDefinitions: make(map[string]error), - } -} - -func (rfl *registrationFailureLog) AddInvalidDefinition(path string, err error){ - rfl.invalidDefinitions[path] = err -} - -func (rfl *registrationFailureLog) InsertEntityRegFailure(hostname string, modelName string,entityType entity.EntityType, name string, err error){ - if(rfl.failureData[hostname] == nil ) { - rfl.failureData[hostname] = make(map[string]map[string]map[string]error) - } - if(rfl.failureData[hostname][modelName] == nil){ - rfl.failureData[hostname][modelName] = make(map[string]map[string]error) - } - if(rfl.failureData[hostname][modelName][string(entityType)] == nil){ - rfl.failureData[hostname][modelName][string(entityType)] = make(map[string]error) - } - rfl.failureData[hostname][modelName][string(entityType)][name] = err -} - -func (rfl *registrationFailureLog) GetNonRegisteredEntitiesCount()map[string]v1beta1.EntitySummary{ - res := make(map[string]v1beta1.EntitySummary) - for registrant, modelNamespacedData := range rfl.failureData { - entitySummary := v1beta1.EntitySummary{} - for modelName, entityTypeNamespacedData := range modelNamespacedData { - if(modelName == ""){ - modelsCount := len(entityTypeNamespacedData[string(entity.Model)]) - entitySummary.Models = int64(modelsCount) - continue - } - entitySummary.Components = int64(len(entityTypeNamespacedData[string(entity.ComponentDefinition)])) - entitySummary.Relationships = int64(len(entityTypeNamespacedData[string(entity.RelationshipDefinition)])) - entitySummary.Policies = int64(len(entityTypeNamespacedData[string(entity.PolicyDefinition)])) - } - res[registrant] = entitySummary - } - return res -} - - -func failedMsgCompute(failedMsg string, hostName string) (string, error) { - nonImportModel := RegLog.GetNonRegisteredEntitiesCount()[hostName] - - if nonImportModel.Models > 0 || nonImportModel.Components > 0 || nonImportModel.Relationships > 0 || nonImportModel.Policies > 0 { - failedMsg = "Failed to import" - appendIfNonZero := func(msg string, count int64, entityName string) string { - if count > 0 { - return fmt.Sprintf("%s %d %s", msg, count, entityName) - } - return msg - } - - failedMsg = appendIfNonZero(failedMsg, nonImportModel.Models, "models") - failedMsg = appendIfNonZero(failedMsg, nonImportModel.Components, "components") - failedMsg = appendIfNonZero(failedMsg, nonImportModel.Relationships, "relationships") - failedMsg = appendIfNonZero(failedMsg, nonImportModel.Policies, "policies") - } - return failedMsg, nil -} - -func FailedEventCompute(hostname string, mesheryInstanceID gofrs.UUID, provider *models.Provider, userID string, ec *models.Broadcast) (string, error) { - - failedMsg, err := failedMsgCompute("", hostname) - if err != nil { - return "", err - } - if failedMsg != "" { - filePath := viper.GetString("REGISTRY_LOG_FILE") - errorEventBuilder := events.NewEvent().FromUser(mesheryInstanceID).FromSystem(mesheryInstanceID).WithCategory("registration").WithAction("get_summary") - errorEventBuilder.WithSeverity(events.Error).WithDescription(failedMsg) - errorEvent := errorEventBuilder.Build() - errorEventBuilder.WithMetadata(map[string]interface{}{ - "DownloadLink": filePath, - "ViewLink": filePath, - "error": ErrImportFailure(hostname, failedMsg), - }) - _ = (*provider).PersistEvent(errorEvent) - if userID != "" { - userUUID := gofrs.FromStringOrNil(userID) - ec.Publish(userUUID, errorEvent) - - } - } - return failedMsg, nil - -} - - -func writeLogsToFiles() error { - filePath := viper.GetString("REGISTRY_LOG_FILE") - // Initialize the formatted log message - var logMessage strings.Builder - - logMessage.WriteString("Invalid Definitions: \n") - for path, err := range RegLog.invalidDefinitions { - logMessage.WriteString(TAB + "[" + path + "] " + err.Error() + " \n" ) - } - logMessage.WriteString("---------------------------------------- \n") - for host, modelNamespacedData := range RegLog.failureData { - logMessage.WriteString(fmt.Sprintf("%s failed to register:\n", host)) - for modelName, entityTypeNamespacedData := range modelNamespacedData { - if(modelName == "") { - logMessage.WriteString(fmt.Sprintf("%sModels:\n", TAB)) - for name, modelData := range entityTypeNamespacedData[string(entity.Model)]{ - logMessage.WriteString(TAB + TAB + name + ": " + modelData.Error()+ "\n") - } - continue - } - logMessage.WriteString(fmt.Sprintf("%sFor Model %s:\n", TAB, modelName)) - for entityType, entityNameNamespacedData := range entityTypeNamespacedData { - logMessage.WriteString(fmt.Sprintf("%s%s%s:\n", TAB,TAB, entityType)) - for name, err := range entityNameNamespacedData { - logMessage.WriteString(fmt.Sprintf("%s%s%s%s: %s\n", TAB,TAB, TAB, name, err.Error() )) - } - - } - } - } - // Write the formatted log message to the file - err := mutils.WriteToFile(filePath, logMessage.String()) - if err != nil { - return meshmodel.ErrWritingRegisteryAttempts(err) - } - return nil -} \ No newline at end of file diff --git a/models/registration/oci.go b/models/registration/oci.go index 75ff41b2..4aa42f50 100644 --- a/models/registration/oci.go +++ b/models/registration/oci.go @@ -10,7 +10,7 @@ type OCIImage struct { _ gcrv1.Image } -func (o OCIImage) PkgUnit() (packagingUnit, error){ +func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error){ pkg := packagingUnit{} return pkg, nil -} \ No newline at end of file +} diff --git a/models/registration/register.go b/models/registration/register.go index 78ac51d7..6971952e 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -1,20 +1,13 @@ package registration import ( - "fmt" - - "github.com/gofrs/uuid" - "github.com/layer5io/meshery/server/models" - "github.com/layer5io/meshkit/logger" - "github.com/layer5io/meshkit/models/events" "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" meshmodel "github.com/layer5io/meshkit/models/meshmodel/registry" - "github.com/spf13/viper" ) -// packaingUnit is the representation of the atomic unit that can be registered into meshery server +// packaingUnit is the representation of the atomic unit that can be registered into the capabilities registry type packagingUnit struct { model v1beta1.Model components []v1beta1.ComponentDefinition @@ -23,28 +16,26 @@ type packagingUnit struct { } type RegistrationHelper struct { - handlerConfig *models.HandlerConfig regManager *meshmodel.RegistryManager - log logger.Handler + regErrStore RegistrationErrorStore } -func NewRegistrationHelper(log logger.Handler, hc *models.HandlerConfig, regm *meshmodel.RegistryManager) RegistrationHelper { - return RegistrationHelper{handlerConfig: hc, log: log, regManager: regm} +func NewRegistrationHelper(regm *meshmodel.RegistryManager, regErrStore RegistrationErrorStore) RegistrationHelper { + return RegistrationHelper{ regManager: regm, regErrStore: regErrStore} } /* Register will accept a RegisterableEntity (dir, tar or oci for now). - Errors are written to the log file. */ -func (rh *RegistrationHelper) Register(entity RegisterableEntity) { +func (rh *RegistrationHelper) Register(entity RegisterableEntity) error { // get the packaging units - pu, err := entity.PkgUnit() + pu, err := entity.PkgUnit(rh.regErrStore) if(err != nil){ // given input is not a valid model, or could not walk the directory - return + return err } // fmt.Printf("Packaging Unit: Model name: %s, comps: %d, rels: %d\n", pu.model.Name, len(pu.components), len(pu.relationships)) - rh.register(pu) + return rh.register(pu) } @@ -52,15 +43,15 @@ func (rh *RegistrationHelper) Register(entity RegisterableEntity) { register will return an error if it is not able to register the `model`. If there are errors when registering other entities, they are handled properly but does not stop the registration process. */ -func (rh *RegistrationHelper)register(pkg packagingUnit) { +func (rh *RegistrationHelper)register(pkg packagingUnit) error { // 1. Register the model model := pkg.model // Dont register anything else if registrant is not there if(model.Registrant.Hostname == ""){ err := ErrMissingRegistrant(model.Name) - RegLog.InsertEntityRegFailure(model.Registrant.Hostname, "",entity.Model, model.Name, err) - return + rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "",entity.Model, model.Name, err) + return err } _, _, err := rh.regManager.RegisterEntity( v1beta1.Host{Hostname: model.Registrant.Hostname,}, @@ -70,8 +61,8 @@ func (rh *RegistrationHelper)register(pkg packagingUnit) { // If model cannot be registered, don't register anything else if err != nil { err = ErrRegisterEntity(err, string(model.Type()), model.DisplayName) - RegLog.InsertEntityRegFailure(model.Registrant.Hostname, "",entity.Model, model.Name, err) - return + rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "",entity.Model, model.Name, err) + return err } hostname := model.Registrant.Hostname @@ -85,8 +76,7 @@ func (rh *RegistrationHelper)register(pkg packagingUnit) { ) if err != nil { err = ErrRegisterEntity(err, string(comp.Type()), comp.DisplayName) - RegLog.InsertEntityRegFailure(hostname, modelName ,entity.ComponentDefinition, comp.DisplayName, err) - rh.log.Error(err) + rh.regErrStore.InsertEntityRegError(hostname, modelName ,entity.ComponentDefinition, comp.DisplayName, err) } } @@ -98,56 +88,8 @@ func (rh *RegistrationHelper)register(pkg packagingUnit) { }, &rel) if err != nil { err = ErrRegisterEntity(err, string(rel.Type()), rel.Kind) - rh.log.Error(err) - RegLog.InsertEntityRegFailure(hostname, modelName ,entity.RelationshipDefinition, rel.ID.String(), err) + rh.regErrStore.InsertEntityRegError(hostname, modelName ,entity.RelationshipDefinition, rel.ID.String(), err) } } + return nil } - -func (erh *RegistrationHelper) RegistryLog() { - log := erh.log - provider := erh.handlerConfig.Providers["None"] - - systemID := viper.GetString("INSTANCE_ID") - - sysID := uuid.FromStringOrNil(systemID) - hosts, _, err := erh.regManager.GetRegistrants(&v1beta1.HostFilter{}) - if err != nil { - log.Error(err) - } - - for _, host := range hosts { - eventBuilder := events.NewEvent().FromSystem(sysID).FromUser(sysID).WithCategory("entity").WithAction("get_summary") - successMessage := fmt.Sprintf("For registrant %s successfully imported", host.Hostname) - appendIfNonZero := func(value int64, label string) { - if value != 0 { - successMessage += fmt.Sprintf(" %d %s", value, label) - } - } - appendIfNonZero(host.Summary.Models, "models") - appendIfNonZero(host.Summary.Components, "components") - appendIfNonZero(host.Summary.Relationships, "relationships") - appendIfNonZero(host.Summary.Policies, "policies") - - log.Info(successMessage) - eventBuilder.WithMetadata(map[string]interface{}{ - "Hostname": host.Hostname, - }) - eventBuilder.WithSeverity(events.Informational).WithDescription(successMessage) - successEvent := eventBuilder.Build() - _ = provider.PersistEvent(successEvent) - - failLog, err := FailedEventCompute(host.Hostname, sysID, &provider, "", erh.handlerConfig.EventBroadcaster) - if err != nil { - log.Error(err) - } - if failLog != "" { - log.Error(meshmodel.ErrRegisteringEntity(failLog, host.Hostname)) - } - - } - err = writeLogsToFiles() - if err != nil { - log.Error(err) - } -} \ No newline at end of file diff --git a/models/registration/tar.go b/models/registration/tar.go index 62625def..66decb59 100644 --- a/models/registration/tar.go +++ b/models/registration/tar.go @@ -4,7 +4,7 @@ type Tar struct { _ string } -func (t Tar) PkgUnit() (packagingUnit, error){ +func (t Tar) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error){ pkg := packagingUnit{} return pkg, nil -} \ No newline at end of file +} diff --git a/models/registration/utils.go b/models/registration/utils.go index 2139d64b..9c96de1a 100644 --- a/models/registration/utils.go +++ b/models/registration/utils.go @@ -54,4 +54,4 @@ func getEntity(byt []byte) (et entity.Entity, _ error) { return nil, ErrGetEntity(fmt.Errorf("Not a valid component definition, model definition, or relationship definition")) } return et, nil -} \ No newline at end of file +} From def3f0040c7496bbc17d32117fb1489f3232533e Mon Sep 17 00:00:00 2001 From: Nithish Karthik Date: Tue, 6 Aug 2024 15:30:07 +0530 Subject: [PATCH 006/112] refactor: Query only models with enabled status Signed-off-by: Nithish Karthik --- models/meshmodel/registry/v1alpha2/relationship_filter.go | 4 ++++ models/meshmodel/registry/v1beta1/component_filter.go | 3 +++ 2 files changed, 7 insertions(+) diff --git a/models/meshmodel/registry/v1alpha2/relationship_filter.go b/models/meshmodel/registry/v1alpha2/relationship_filter.go index 64f148fa..eed6b171 100644 --- a/models/meshmodel/registry/v1alpha2/relationship_filter.go +++ b/models/meshmodel/registry/v1alpha2/relationship_filter.go @@ -48,6 +48,10 @@ func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entit finder := db.Model(&v1alpha2.RelationshipDefinition{}).Preload("Model").Preload("Model.Category"). Joins("JOIN model_dbs ON relationship_definition_dbs.model_id = model_dbs.id"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") + + // TODO(@MUzairS15): Refactor this once Status is made a first class field in RelationshipFilter + finder = finder.Where("model_dbs.status = enabled") + if relationshipFilter.Kind != "" { if relationshipFilter.Greedy { finder = finder.Where("relationship_definition_dbs.kind LIKE ?", "%"+relationshipFilter.Kind+"%") diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index 72f0f9c5..24ddb555 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -69,6 +69,9 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti Joins("JOIN hosts ON hosts.id = model_dbs.host_id") // + // TODO(@MUzairS15): Refactor this once Status is made a first class field in ComponentFilter + finder = finder.Where("model_dbs.status = enabled") + if componentFilter.Greedy { if componentFilter.Name != "" && componentFilter.DisplayName != "" { finder = finder.Where("component_definition_dbs.component->>'kind' LIKE ? OR display_name LIKE ?", "%"+componentFilter.Name+"%", componentFilter.DisplayName+"%") From a5da4b01ef5965ece87ebf9061af46406d929c6a Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 6 Aug 2024 15:52:43 +0530 Subject: [PATCH 007/112] minor update: Signed-off-by: MUzairS15 --- .../registry/v1alpha2/relationship_filter.go | 19 +++++++++++++------ .../registry/v1beta1/component_filter.go | 19 +++++++++++++------ .../registry/v1beta1/model_filter.go | 9 +++++++++ 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/models/meshmodel/registry/v1alpha2/relationship_filter.go b/models/meshmodel/registry/v1alpha2/relationship_filter.go index eed6b171..b9ac33ab 100644 --- a/models/meshmodel/registry/v1alpha2/relationship_filter.go +++ b/models/meshmodel/registry/v1alpha2/relationship_filter.go @@ -12,7 +12,7 @@ import ( // In the future, we will add support to query using `selectors` (using CUE) // TODO: Add support for Model type RelationshipFilter struct { - Id string + Id string Kind string Greedy bool //when set to true - instead of an exact match, kind will be prefix matched SubType string @@ -23,6 +23,7 @@ type RelationshipFilter struct { Sort string //asc or desc. Default behavior is asc Limit int //If 0 or unspecified then all records are returned and limit is not used Offset int + Status string } // Create the filter from map[string]interface{} @@ -34,12 +35,12 @@ func (rf *RelationshipFilter) Create(m map[string]interface{}) { } func (rf *RelationshipFilter) GetById(db *database.Handler) (entity.Entity, error) { - r := &v1alpha2.RelationshipDefinition{} - err := db.First(r, "id = ?", rf.Id).Error + r := &v1alpha2.RelationshipDefinition{} + err := db.First(r, "id = ?", rf.Id).Error if err != nil { return nil, registry.ErrGetById(err, rf.Id) } - return r, err + return r, err } func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { @@ -49,8 +50,14 @@ func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entit Joins("JOIN model_dbs ON relationship_definition_dbs.model_id = model_dbs.id"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") - // TODO(@MUzairS15): Refactor this once Status is made a first class field in RelationshipFilter - finder = finder.Where("model_dbs.status = enabled") + // TODO(@MUzairS15): Refactor this once Status is made a first class field in RelationshipFilter + status := "enabled" + + if relationshipFilter.Status != "" { + status = relationshipFilter.Status + } + + finder = finder.Where("model_dbs.status = ?", status) if relationshipFilter.Kind != "" { if relationshipFilter.Greedy { diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index 24ddb555..c6996cda 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -9,7 +9,7 @@ import ( ) type ComponentFilter struct { - Id string + Id string Name string APIVersion string Greedy bool //when set to true - instead of an exact match, name will be matched as a substring @@ -23,6 +23,7 @@ type ComponentFilter struct { Limit int //If 0 or unspecified then all records are returned and limit is not used Offset int Annotations string //When this query parameter is "true", only components with the "isAnnotation" property set to true are returned. When this query parameter is "false", all components except those considered to be annotation components are returned. Any other value of the query parameter results in both annotations as well as non-annotation models being returned. + Status string } type componentDefinitionWithModel struct { @@ -33,12 +34,12 @@ type componentDefinitionWithModel struct { } func (cf *ComponentFilter) GetById(db *database.Handler) (entity.Entity, error) { - c := &v1beta1.ComponentDefinition{} - err := db.First(c, "id = ?", cf.Id).Error + c := &v1beta1.ComponentDefinition{} + err := db.First(c, "id = ?", cf.Id).Error if err != nil { return nil, registry.ErrGetById(err, cf.Id) } - return c, err + return c, err } // Create the filter from map[string]interface{} @@ -69,8 +70,14 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti Joins("JOIN hosts ON hosts.id = model_dbs.host_id") // - // TODO(@MUzairS15): Refactor this once Status is made a first class field in ComponentFilter - finder = finder.Where("model_dbs.status = enabled") + // TODO(@MUzairS15): Refactor this once Status is made a first class field in ComponentFilter + status := "enabled" + + if componentFilter.Status != "" { + status = componentFilter.Status + } + + finder = finder.Where("model_dbs.status = ?", status) if componentFilter.Greedy { if componentFilter.Name != "" && componentFilter.DisplayName != "" { diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index 440ecb3e..d8969671 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -127,6 +127,15 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e if mf.Offset != 0 { finder = finder.Offset(mf.Offset) } + + status := "enabled" + + if mf.Status != "" { + status = mf.Status + } + + finder = finder.Where("model_dbs.status = ?", status) + if mf.Status != "" { finder = finder.Where("model_dbs.status = ?", mf.Status) } From 884059ba86eafd1b17a058ba4247ba69fcd4ba47 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Wed, 7 Aug 2024 11:12:34 +0530 Subject: [PATCH 008/112] fix linting w/signoff Signed-off-by: Jougan-0 --- models/oci/error.go | 4 ++++ models/oci/utils.go | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/models/oci/error.go b/models/oci/error.go index f3e51c77..307e43c9 100644 --- a/models/oci/error.go +++ b/models/oci/error.go @@ -22,6 +22,7 @@ var ( ErrAddLayerCode = "meshkit-11247" ErrTaggingPackageCode = "meshkit-11248" ErrPushingPackageCode = "meshkit-11249" + ErrSeekFailedCode = "replace_me" ) func ErrAppendingLayer(err error) error { @@ -83,3 +84,6 @@ func ErrTaggingPackage(err error) error { func ErrPushingPackage(err error) error { return errors.New(ErrPushingPackageCode, errors.Alert, []string{"pushing package failed"}, []string{err.Error()}, []string{"failed to push the package"}, []string{"Try using a different tag", "check if package is not malformed"}) } +func ErrSeekFailed(err error) error { + return errors.New(ErrSeekFailedCode, errors.Alert, []string{"Unable to reset the position within the OCI data."}, []string{err.Error()}, []string{"The function attempted to move to the start of the data but failed. This could happen if the data is corrupted or not in the expected format."}, []string{"Ensure the input data is a valid OCI archive and try again. Check if the data is compressed correctly and is not corrupted."}) +} diff --git a/models/oci/utils.go b/models/oci/utils.go index 99cc1002..69fd31c9 100644 --- a/models/oci/utils.go +++ b/models/oci/utils.go @@ -89,7 +89,10 @@ func IsOCIArtifact(data []byte) bool { defer gzr.Close() tr = archiveTar.NewReader(gzr) } else { - reader.Seek(0, io.SeekStart) + _, err := reader.Seek(0, io.SeekStart) + if err != nil { + return false + } tr = archiveTar.NewReader(reader) } @@ -118,7 +121,10 @@ func ExtractAndValidateManifest(data []byte) error { defer gzr.Close() tr = archiveTar.NewReader(gzr) } else { - reader.Seek(0, io.SeekStart) + _, err := reader.Seek(0, io.SeekStart) + if err != nil { + return ErrSeekFailed(err) + } tr = archiveTar.NewReader(reader) } From a74edb0f506401fe9894732b128ac11563c3cded Mon Sep 17 00:00:00 2001 From: Nithish Karthik Date: Wed, 7 Aug 2024 18:13:17 +0530 Subject: [PATCH 009/112] refactor: Do not return errors Signed-off-by: Nithish Karthik --- models/registration/interface.go | 2 +- models/registration/register.go | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/models/registration/interface.go b/models/registration/interface.go index a7ee0732..c13f1394 100644 --- a/models/registration/interface.go +++ b/models/registration/interface.go @@ -15,7 +15,7 @@ type RegistrationErrorStore interface { // Anything that can be parsed into a packagingUnit is a RegisterableEntity in Meshery server type RegisterableEntity interface { /* - 1. `err` - this is a breaking error, which signifies that the given entity is invalid + 1. `err` - this is a breaking error, which signifies that the given entity is invalid and cannot be registered 2. Errors encountered while parsing items into meshmodel entites are stored in the RegistrationErrorStore */ PkgUnit(RegistrationErrorStore) (packagingUnit, error) diff --git a/models/registration/register.go b/models/registration/register.go index 6971952e..0e2478c7 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -27,23 +27,21 @@ func NewRegistrationHelper(regm *meshmodel.RegistryManager, regErrStore Registra /* Register will accept a RegisterableEntity (dir, tar or oci for now). */ -func (rh *RegistrationHelper) Register(entity RegisterableEntity) error { +func (rh *RegistrationHelper) Register(entity RegisterableEntity) { // get the packaging units pu, err := entity.PkgUnit(rh.regErrStore) if(err != nil){ // given input is not a valid model, or could not walk the directory - return err + return } - // fmt.Printf("Packaging Unit: Model name: %s, comps: %d, rels: %d\n", pu.model.Name, len(pu.components), len(pu.relationships)) - return rh.register(pu) + rh.register(pu) } - /* register will return an error if it is not able to register the `model`. If there are errors when registering other entities, they are handled properly but does not stop the registration process. */ -func (rh *RegistrationHelper)register(pkg packagingUnit) error { +func (rh *RegistrationHelper)register(pkg packagingUnit) { // 1. Register the model model := pkg.model @@ -51,7 +49,7 @@ func (rh *RegistrationHelper)register(pkg packagingUnit) error { if(model.Registrant.Hostname == ""){ err := ErrMissingRegistrant(model.Name) rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "",entity.Model, model.Name, err) - return err + return } _, _, err := rh.regManager.RegisterEntity( v1beta1.Host{Hostname: model.Registrant.Hostname,}, @@ -62,7 +60,7 @@ func (rh *RegistrationHelper)register(pkg packagingUnit) error { if err != nil { err = ErrRegisterEntity(err, string(model.Type()), model.DisplayName) rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "",entity.Model, model.Name, err) - return err + return } hostname := model.Registrant.Hostname @@ -91,5 +89,4 @@ func (rh *RegistrationHelper)register(pkg packagingUnit) error { rh.regErrStore.InsertEntityRegError(hostname, modelName ,entity.RelationshipDefinition, rel.ID.String(), err) } } - return nil } From 091910c2c7e714d87005aa6f68d7f998dffc09ac Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 7 Aug 2024 19:01:30 +0530 Subject: [PATCH 010/112] run gofmt Signed-off-by: MUzairS15 --- models/registration/register.go | 49 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/models/registration/register.go b/models/registration/register.go index 6971952e..4960fd5f 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -9,28 +9,28 @@ import ( // packaingUnit is the representation of the atomic unit that can be registered into the capabilities registry type packagingUnit struct { - model v1beta1.Model - components []v1beta1.ComponentDefinition + model v1beta1.Model + components []v1beta1.ComponentDefinition relationships []v1alpha2.RelationshipDefinition - _ []v1beta1.PolicyDefinition + _ []v1beta1.PolicyDefinition } type RegistrationHelper struct { - regManager *meshmodel.RegistryManager - regErrStore RegistrationErrorStore + regManager *meshmodel.RegistryManager + regErrStore RegistrationErrorStore } func NewRegistrationHelper(regm *meshmodel.RegistryManager, regErrStore RegistrationErrorStore) RegistrationHelper { - return RegistrationHelper{ regManager: regm, regErrStore: regErrStore} + return RegistrationHelper{regManager: regm, regErrStore: regErrStore} } /* - Register will accept a RegisterableEntity (dir, tar or oci for now). +Register will accept a RegisterableEntity (dir, tar or oci for now). */ func (rh *RegistrationHelper) Register(entity RegisterableEntity) error { // get the packaging units pu, err := entity.PkgUnit(rh.regErrStore) - if(err != nil){ + if err != nil { // given input is not a valid model, or could not walk the directory return err } @@ -38,30 +38,29 @@ func (rh *RegistrationHelper) Register(entity RegisterableEntity) error { return rh.register(pu) } - /* - register will return an error if it is not able to register the `model`. - If there are errors when registering other entities, they are handled properly but does not stop the registration process. +register will return an error if it is not able to register the `model`. +If there are errors when registering other entities, they are handled properly but does not stop the registration process. */ -func (rh *RegistrationHelper)register(pkg packagingUnit) error { +func (rh *RegistrationHelper) register(pkg packagingUnit) error { // 1. Register the model model := pkg.model // Dont register anything else if registrant is not there - if(model.Registrant.Hostname == ""){ + if model.Registrant.Hostname == "" { err := ErrMissingRegistrant(model.Name) - rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "",entity.Model, model.Name, err) + rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "", entity.Model, model.Name, err) return err } _, _, err := rh.regManager.RegisterEntity( - v1beta1.Host{Hostname: model.Registrant.Hostname,}, + v1beta1.Host{Hostname: model.Registrant.Hostname}, &model, - ) + ) // If model cannot be registered, don't register anything else if err != nil { err = ErrRegisterEntity(err, string(model.Type()), model.DisplayName) - rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "",entity.Model, model.Name, err) + rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "", entity.Model, model.Name, err) return err } @@ -71,13 +70,13 @@ func (rh *RegistrationHelper)register(pkg packagingUnit) error { for _, comp := range pkg.components { comp.Model = model _, _, err := rh.regManager.RegisterEntity( - v1beta1.Host{Hostname: hostname,}, - &comp, + v1beta1.Host{Hostname: hostname}, + &comp, ) - if err != nil { - err = ErrRegisterEntity(err, string(comp.Type()), comp.DisplayName) - rh.regErrStore.InsertEntityRegError(hostname, modelName ,entity.ComponentDefinition, comp.DisplayName, err) - } + if err != nil { + err = ErrRegisterEntity(err, string(comp.Type()), comp.DisplayName) + rh.regErrStore.InsertEntityRegError(hostname, modelName, entity.ComponentDefinition, comp.DisplayName, err) + } } // 3. Register relationships @@ -88,8 +87,8 @@ func (rh *RegistrationHelper)register(pkg packagingUnit) error { }, &rel) if err != nil { err = ErrRegisterEntity(err, string(rel.Type()), rel.Kind) - rh.regErrStore.InsertEntityRegError(hostname, modelName ,entity.RelationshipDefinition, rel.ID.String(), err) + rh.regErrStore.InsertEntityRegError(hostname, modelName, entity.RelationshipDefinition, rel.ID.String(), err) } } - return nil + return nil } From 95ea8a013d15694daf85ce1b4600e41423451c83 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 7 Aug 2024 19:24:53 +0530 Subject: [PATCH 011/112] write svg to file system Signed-off-by: MUzairS15 --- .../registry/v1beta1/model_filter.go | 4 - models/registration/register.go | 7 +- models/registration/svg_helper.go | 134 ++++++++++++++++++ 3 files changed, 139 insertions(+), 6 deletions(-) create mode 100644 models/registration/svg_helper.go diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index d8969671..d4364c89 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -136,10 +136,6 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e finder = finder.Where("model_dbs.status = ?", status) - if mf.Status != "" { - finder = finder.Where("model_dbs.status = ?", mf.Status) - } - includeComponents = mf.Components includeRelationships = mf.Relationships diff --git a/models/registration/register.go b/models/registration/register.go index 4960fd5f..3c8de734 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -18,10 +18,11 @@ type packagingUnit struct { type RegistrationHelper struct { regManager *meshmodel.RegistryManager regErrStore RegistrationErrorStore + svgBaseDir string } -func NewRegistrationHelper(regm *meshmodel.RegistryManager, regErrStore RegistrationErrorStore) RegistrationHelper { - return RegistrationHelper{regManager: regm, regErrStore: regErrStore} +func NewRegistrationHelper(svgBaseDir string, regm *meshmodel.RegistryManager, regErrStore RegistrationErrorStore) RegistrationHelper { + return RegistrationHelper{svgBaseDir: svgBaseDir, regManager: regm, regErrStore: regErrStore} } /* @@ -52,6 +53,7 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) error { rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "", entity.Model, model.Name, err) return err } + writeAndReplaceSVGWithFileSystemPath(model.Metadata, rh.svgBaseDir, model.Name, model.Name) //Write SVG for models _, _, err := rh.regManager.RegisterEntity( v1beta1.Host{Hostname: model.Registrant.Hostname}, &model, @@ -69,6 +71,7 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) error { // 2. Register components for _, comp := range pkg.components { comp.Model = model + writeAndReplaceSVGWithFileSystemPath(comp.Metadata, rh.svgBaseDir, comp.Model.Name, comp.Component.Kind) //Write SVG on components _, _, err := rh.regManager.RegisterEntity( v1beta1.Host{Hostname: hostname}, &comp, diff --git a/models/registration/svg_helper.go b/models/registration/svg_helper.go new file mode 100644 index 00000000..8bcb75ad --- /dev/null +++ b/models/registration/svg_helper.go @@ -0,0 +1,134 @@ +package registration + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + "sync" +) + +var hashCheckSVG = make(map[string]string) +var mx sync.Mutex +var UISVGPaths = make([]string, 1) + +func writeHashCheckSVG(key string, val string) { + mx.Lock() + hashCheckSVG[key] = val + mx.Unlock() +} + +func writeAndReplaceSVGWithFileSystemPath(metadata map[string]interface{}, baseDir, dirname, filename string) { + filename = strings.ToLower(filename) + successCreatingDirectory := false + defer func() { + if successCreatingDirectory { + UISVGPaths = append(UISVGPaths, filepath.Join(baseDir, dirname)) + } + }() + if metadata["svgColor"] != "" { + path := filepath.Join(baseDir, dirname, "color") + err := os.MkdirAll(path, 0777) + if err != nil { + fmt.Println(err) + return + } + successCreatingDirectory = true + + x, ok := metadata["svgColor"].(string) + if ok { + hash := md5.Sum([]byte(x)) + hashString := hex.EncodeToString(hash[:]) + pathsvg := hashCheckSVG[hashString] + if pathsvg != "" { // the image has already been loaded, point the component to that path + metadata["svgColor"] = pathsvg + goto White + } + f, err := os.Create(filepath.Join(path, filename+"-color.svg")) + if err != nil { + fmt.Println(err) + return + } + _, err = f.WriteString(x) + if err != nil { + fmt.Println(err) + return + } + metadata["svgColor"] = getRelativePathForAPI(baseDir, filepath.Join(dirname, "color", filename+"-color.svg")) //Replace the actual SVG with path to SVG + writeHashCheckSVG(hashString, metadata["svgColor"].(string)) + } + } +White: + if metadata["svgWhite"] != "" { + path := filepath.Join(baseDir, dirname, "white") + err := os.MkdirAll(path, 0777) + if err != nil { + fmt.Println(err) + return + } + successCreatingDirectory = true + + x, ok := metadata["svgWhite"].(string) + if ok { + hash := md5.Sum([]byte(x)) + hashString := hex.EncodeToString(hash[:]) + pathsvg := hashCheckSVG[hashString] + if pathsvg != "" { // the image has already been loaded, point the component to that path + metadata["svgWhite"] = pathsvg + goto Complete + } + f, err := os.Create(filepath.Join(path, filename+"-white.svg")) + if err != nil { + fmt.Println(err) + return + } + _, err = f.WriteString(x) + if err != nil { + fmt.Println(err) + return + } + metadata["svgWhite"] = getRelativePathForAPI(baseDir, filepath.Join(dirname, "white", filename+"-white.svg")) //Replace the actual SVG with path to SVG + writeHashCheckSVG(hashString, metadata["svgWhite"].(string)) + } + } +Complete: + if metadata["svgComplete"] != "" { + path := filepath.Join(baseDir, dirname, "complete") + err := os.MkdirAll(path, 0777) + if err != nil { + fmt.Println(err) + return + } + successCreatingDirectory = true + + x, ok := metadata["svgComplete"].(string) + if ok { + hash := md5.Sum([]byte(x)) + hashString := hex.EncodeToString(hash[:]) + pathsvg := hashCheckSVG[hashString] + if pathsvg != "" { // the image has already been loaded, point the component to that path + metadata["svgComplete"] = pathsvg + return + } + f, err := os.Create(filepath.Join(path, filename+"-complete.svg")) + if err != nil { + fmt.Println(err) + return + } + _, err = f.WriteString(x) + if err != nil { + fmt.Println(err) + return + } + metadata["svgComplete"] = getRelativePathForAPI(baseDir, filepath.Join(dirname, "complete", filename+"-complete.svg")) //Replace the actual SVG with path to SVG + writeHashCheckSVG(hashString, metadata["svgComplete"].(string)) + } + } +} + +func getRelativePathForAPI(baseDir, path string) string { + ui := strings.TrimPrefix(baseDir, "../../") + return filepath.Join(ui, path) +} From e706e484155147af8fe433903ee1cb1fb7bc896d Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 7 Aug 2024 19:27:55 +0530 Subject: [PATCH 012/112] remove static status Signed-off-by: MUzairS15 --- models/meshmodel/core/v1beta1/models.go | 1 - 1 file changed, 1 deletion(-) diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index e3094f6c..0fc27a27 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -97,7 +97,6 @@ func (m *Model) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error m.ID = modelID m.CategoryID = id m.RegistrantID = hostID - m.Status = entity.Enabled err = db.Omit(clause.Associations).Create(&m).Error if err != nil { return uuid.UUID{}, err From 86b7fb93b7ba511103646e01c0551de85b72bba1 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Thu, 8 Aug 2024 23:31:33 +0530 Subject: [PATCH 013/112] fix versionmeta and missing schemaversion w/signoff Signed-off-by: Jougan-0 --- .../meshmodel/core/v1alpha2/relationship.go | 6 +-- models/meshmodel/core/v1beta1/component.go | 12 +++--- models/meshmodel/core/v1beta1/models.go | 5 ++- .../registry/v1beta1/model_filter.go | 42 +++++++++++++++---- 4 files changed, 45 insertions(+), 20 deletions(-) diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go index 1b10bf86..931ef10f 100644 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -16,9 +16,9 @@ import ( const RelationshipSchemaVersion = "relationships.meshery.io/v1alpha2" type RelationshipDefinition struct { - ID uuid.UUID `json:"id"` - v1beta1.VersionMeta - Kind string `json:"kind,omitempty" yaml:"kind"` + ID uuid.UUID `json:"id"` + v1beta1.VersionMeta `json:",inline" yaml:",inline"` + Kind string `json:"kind,omitempty" yaml:"kind"` // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. // Though, the column name and the json representation is "type". RelationshipType string `json:"type" yaml:"type" gorm:"type"` diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index fbb56b7e..0722ec41 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -13,8 +13,8 @@ import ( ) type VersionMeta struct { - SchemaVersion string `json:"schemaVersion,omitempty" yaml:"schemaVersion"` - Version string `json:"version,omitempty" yaml:"version"` + SchemaVersion string `json:"schemaVersion,omitempty" yaml:"schemaVersion" gorm:"column:schema_version"` + Version string `json:"version,omitempty" yaml:"version" gorm:"column:version"` } type TypeMeta struct { @@ -40,10 +40,10 @@ type ComponentEntity struct { // swagger:response ComponentDefinition type ComponentDefinition struct { - ID uuid.UUID `json:"id"` - VersionMeta - DisplayName string `json:"displayName" gorm:"displayName"` - Description string `json:"description" gorm:"description"` + ID uuid.UUID `json:"id"` + VersionMeta `json:",inline" yaml:",inline"` + DisplayName string `json:"displayName" gorm:"column:display_name"` + Description string `json:"description" gorm:"column:description"` Format ComponentFormat `json:"format" yaml:"format"` ModelID uuid.UUID `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:model_id"` Model Model `json:"model" gorm:"foreignKey:ModelID;references:ID"` diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index 0fc27a27..b56ee887 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -24,8 +24,9 @@ type ModelEntity struct { // swagger:response Model type Model struct { - ID uuid.UUID `json:"id"` - VersionMeta + ID uuid.UUID `json:"id"` + VersionMeta `yaml:",inline"` + Name string `json:"name" gorm:"modelName"` DisplayName string `json:"displayName"` Description string `json:"description" gorm:"description"` diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index d4364c89..d5d3cce2 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -10,7 +10,7 @@ import ( ) type ModelFilter struct { - Id string + Id string Name string Registrant string //name of the registrant for a given model DisplayName string //If Name is already passed, avoid passing Display name unless greedy=true, else the filter will translate to an AND returning only the models where name and display name match exactly. Ignore, if this behavior is expected. @@ -49,18 +49,42 @@ func countUniqueModels(models []v1beta1.Model) int { } func (mf *ModelFilter) GetById(db *database.Handler) (entity.Entity, error) { - m := &v1beta1.Model{} - err := db.First(m, "id = ?", mf.Id).Error + m := &v1beta1.Model{} + // Retrieve the model by ID + err := db.First(m, "id = ?", mf.Id).Error if err != nil { return nil, registry.ErrGetById(err, mf.Id) } - return m, err -} + // Include components if requested + if mf.Components { + var components []v1beta1.ComponentDefinition + componentFinder := db.Model(&v1beta1.ComponentDefinition{}). + Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version"). + Where("component_definition_dbs.model_id = ?", m.ID) + if err := componentFinder.Scan(&components).Error; err != nil { + return nil, err + } + m.Components = components + } -func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { + // Include relationships if requested + if mf.Relationships { + var relationships []v1alpha2.RelationshipDefinition + relationshipFinder := db.Model(&v1alpha2.RelationshipDefinition{}). + Select("relationship_definition_dbs.*"). + Where("relationship_definition_dbs.model_id = ?", m.ID) + if err := relationshipFinder.Scan(&relationships).Error; err != nil { + return nil, err + } + m.Relationships = relationships + } + + return m, nil +} +func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { var modelWithCategories []v1beta1.Model finder := db.Model(&v1beta1.Model{}).Preload("Category").Preload("Registrant"). @@ -129,8 +153,8 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e } status := "enabled" - - if mf.Status != "" { + + if mf.Status != "" { status = mf.Status } @@ -153,7 +177,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e if includeComponents { var components []v1beta1.ComponentDefinition finder := db.Model(&v1beta1.ComponentDefinition{}). - Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata"). + Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version"). Where("component_definition_dbs.model_id = ?", _modelDB.ID) if err := finder.Scan(&components).Error; err != nil { return nil, 0, 0, err From 81f1bd9fc3873ce35bb61ecb2c74b505108740df Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Thu, 8 Aug 2024 23:45:18 +0530 Subject: [PATCH 014/112] meshkit errors w/signoff Signed-off-by: Jougan-0 --- models/oci/error.go | 24 ++++++++++++++++++++++-- models/oci/oci.go | 2 +- models/oci/utils.go | 2 +- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/models/oci/error.go b/models/oci/error.go index 307e43c9..641e49f0 100644 --- a/models/oci/error.go +++ b/models/oci/error.go @@ -23,12 +23,23 @@ var ( ErrTaggingPackageCode = "meshkit-11248" ErrPushingPackageCode = "meshkit-11249" ErrSeekFailedCode = "replace_me" + ErrCreateLayerCode = "replace_me" + ErrSavingImageCode = "replace_me" ) func ErrAppendingLayer(err error) error { return errors.New(ErrAppendingLayerCode, errors.Alert, []string{"appending content to artifact failed"}, []string{err.Error()}, []string{"layer is not compatible with the base image"}, []string{"Try using a different base image", "use a different media type for the layer"}) } - +func ErrSavingImage(err error) error { + return errors.New( + ErrSavingImageCode, + errors.Alert, + []string{"Failed to save image"}, + []string{"An error occurred while saving the image."}, + []string{"Possible issues with file system, insufficient disk space, , other IO errors."}, + []string{"Check the file system permissions and available disk space.", "Ensure the file path is correct and accessible.", "Check for any underlying IO errors."}, + ) +} func ErrReadingFile(err error) error { return errors.New(ErrReadingFileCode, errors.Alert, []string{"reading file failed"}, []string{err.Error()}, []string{"failed to read the file", "Insufficient permissions"}, []string{"Try using a different file", "check if appropriate read permissions are given to the file"}) } @@ -44,7 +55,16 @@ func ErrGettingLayer(err error) error { func ErrCompressingLayer(err error) error { return errors.New(ErrCompressingLayerCode, errors.Alert, []string{"compressing layer failed"}, []string{err.Error()}, []string{"failed to compress the layer"}, []string{"Try using a different layer", "check if layers are compatible with the base image"}) } - +func ErrCreateLayer(err error) error { + return errors.New( + ErrCreateLayerCode, + errors.Alert, + []string{"Failed to create layer"}, + []string{"An error occurred while creating the layer for the image."}, + []string{"Possible issues with file system, incorrect layer type, or other IO errors."}, + []string{"Check the file system permissions and paths.", "Ensure the layer type is correct.", "Check for any underlying IO errors."}, + ) +} func ErrUnTaringLayer(err error) error { return errors.New(ErrUnTaringLayerCode, errors.Alert, []string{"untaring layer failed"}, []string{err.Error()}, []string{"failed to untar the layer"}, []string{"Try using a different layer", "check if image is not malformed"}) } diff --git a/models/oci/oci.go b/models/oci/oci.go index 42e8f525..cde3fec4 100644 --- a/models/oci/oci.go +++ b/models/oci/oci.go @@ -64,7 +64,7 @@ func BuildImage(sourcePath string, opts ...BuildOption) (gcrv1.Image, error) { layer, err := createLayer(sourcePath, o.layerType, o.layerOpts) if err != nil { - return nil, err + return nil, ErrCreateLayer(err) } if o.meta.Created == "" { diff --git a/models/oci/utils.go b/models/oci/utils.go index 69fd31c9..796324bd 100644 --- a/models/oci/utils.go +++ b/models/oci/utils.go @@ -34,7 +34,7 @@ func SaveOCIArtifact(img gcrv1.Image, artifactPath, name string) error { repoWithTag := fmt.Sprintf("%s:%s", name, "latest") // TODO: Add support to make this dynamic from user input err := crane.Save(img, repoWithTag, artifactPath) if err != nil { - return err + return ErrSavingImage(err) } return nil From 1b1fd8b64ede84960272c3403b7570965f6494c4 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Fri, 9 Aug 2024 18:05:57 +0530 Subject: [PATCH 015/112] make WriteAndReplaceSVGWithFileSystemPath global w/signoff Signed-off-by: Jougan-0 --- models/registration/register.go | 10 +++++----- models/registration/svg_helper.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/models/registration/register.go b/models/registration/register.go index 4847d4ce..8f35e8ba 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -42,17 +42,17 @@ func (rh *RegistrationHelper) Register(entity RegisterableEntity) { register will return an error if it is not able to register the `model`. If there are errors when registering other entities, they are handled properly but does not stop the registration process. */ -func (rh *RegistrationHelper)register(pkg packagingUnit) { +func (rh *RegistrationHelper) register(pkg packagingUnit) { // 1. Register the model model := pkg.model // Dont register anything else if registrant is not there if model.Registrant.Hostname == "" { err := ErrMissingRegistrant(model.Name) - rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "",entity.Model, model.Name, err) + rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "", entity.Model, model.Name, err) return } - writeAndReplaceSVGWithFileSystemPath(model.Metadata, rh.svgBaseDir, model.Name, model.Name) //Write SVG for models + WriteAndReplaceSVGWithFileSystemPath(model.Metadata, rh.svgBaseDir, model.Name, model.Name) //Write SVG for models _, _, err := rh.regManager.RegisterEntity( v1beta1.Host{Hostname: model.Registrant.Hostname}, &model, @@ -61,7 +61,7 @@ func (rh *RegistrationHelper)register(pkg packagingUnit) { // If model cannot be registered, don't register anything else if err != nil { err = ErrRegisterEntity(err, string(model.Type()), model.DisplayName) - rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "",entity.Model, model.Name, err) + rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "", entity.Model, model.Name, err) return } @@ -70,7 +70,7 @@ func (rh *RegistrationHelper)register(pkg packagingUnit) { // 2. Register components for _, comp := range pkg.components { comp.Model = model - writeAndReplaceSVGWithFileSystemPath(comp.Metadata, rh.svgBaseDir, comp.Model.Name, comp.Component.Kind) //Write SVG on components + WriteAndReplaceSVGWithFileSystemPath(comp.Metadata, rh.svgBaseDir, comp.Model.Name, comp.Component.Kind) //Write SVG on components _, _, err := rh.regManager.RegisterEntity( v1beta1.Host{Hostname: hostname}, &comp, diff --git a/models/registration/svg_helper.go b/models/registration/svg_helper.go index 8bcb75ad..88f0b35a 100644 --- a/models/registration/svg_helper.go +++ b/models/registration/svg_helper.go @@ -20,7 +20,7 @@ func writeHashCheckSVG(key string, val string) { mx.Unlock() } -func writeAndReplaceSVGWithFileSystemPath(metadata map[string]interface{}, baseDir, dirname, filename string) { +func WriteAndReplaceSVGWithFileSystemPath(metadata map[string]interface{}, baseDir, dirname, filename string) { filename = strings.ToLower(filename) successCreatingDirectory := false defer func() { From 010e2b5970ee4f0a0f1e91e643e46391fd4236ff Mon Sep 17 00:00:00 2001 From: Chris C Date: Fri, 9 Aug 2024 12:01:14 -0400 Subject: [PATCH 016/112] Add default values for required fields in artifacthub-pkg Signed-off-by: Chris C --- utils/catalog/package.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/utils/catalog/package.go b/utils/catalog/package.go index c0860fc4..2438dfcf 100644 --- a/utils/catalog/package.go +++ b/utils/catalog/package.go @@ -4,6 +4,7 @@ import ( "fmt" "regexp" "strings" + "github.com/layer5io/meshkit/models/catalog/v1alpha1" ) @@ -11,7 +12,7 @@ func BuildArtifactHubPkg(name, downloadURL, user, version, createdAt string, cat artifacthubPkg := &ArtifactHubMetadata{ Name: toKebabCase(name), DisplayName: name, - Description: catalogData.PatternInfo, + Description: valueOrElse(catalogData.PatternInfo, "A Meshery Design"), Provider: Provider{ Name: user, }, @@ -26,7 +27,7 @@ func BuildArtifactHubPkg(name, downloadURL, user, version, createdAt string, cat }, }, HomeURL: "https://docs.meshery.io/concepts/logical/designs", - Version: version, + Version: valueOrElse(version, "0.0.1"), CreatedAt: createdAt, License: "Apache-2.0", LogoURL: "https://raw.githubusercontent.com/meshery/meshery.io/0b8585231c6e2b3251d38f749259360491c9ee6b/assets/images/brand/meshery-logo.svg", @@ -56,6 +57,14 @@ func BuildArtifactHubPkg(name, downloadURL, user, version, createdAt string, cat return artifacthubPkg } +func valueOrElse(s string, fallback string) string { + if len(s) == 0 { + return fallback + } else { + return s + } +} + func toKebabCase(s string) string { s = strings.ToLower(s) re := regexp.MustCompile(`\s+`) From 7b533c4d28c6ed61e9abac0d66f5936cdaa6ef32 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 30 Jul 2024 13:21:47 +0530 Subject: [PATCH 017/112] move to connection instead of host Signed-off-by: MUzairS15 --- generators/artifacthub/package.go | 12 +- generators/github/package.go | 12 +- generators/models/interfaces.go | 3 +- go.mod | 2 + go.sum | 2 - .../meshmodel/core/v1alpha2/relationship.go | 16 +- .../core/v1beta1/application_component.go | 104 +++++------- models/meshmodel/core/v1beta1/category.go | 83 ---------- models/meshmodel/core/v1beta1/component.go | 109 ------------- models/meshmodel/core/v1beta1/host.go | 100 +++--------- models/meshmodel/core/v1beta1/models.go | 154 ------------------ models/meshmodel/core/v1beta1/policy.go | 22 +-- models/meshmodel/entity/types.go | 2 +- models/meshmodel/registry/registry.go | 48 +++--- .../registry/v1beta1/category_filter.go | 5 +- .../registry/v1beta1/component_filter.go | 11 +- .../registry/v1beta1/model_filter.go | 13 +- utils/component/generator.go | 11 +- utils/component/generator_test.go | 2 +- validator/validate_test.go | 16 +- 20 files changed, 151 insertions(+), 576 deletions(-) delete mode 100644 models/meshmodel/core/v1beta1/category.go delete mode 100644 models/meshmodel/core/v1beta1/component.go delete mode 100644 models/meshmodel/core/v1beta1/models.go diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index 00bb416d..15c2332f 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/meshery/schemas/models/v1beta1" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" "github.com/layer5io/meshkit/utils/manifests" @@ -53,15 +53,15 @@ func (pkg AhPackage) GenerateComponents() ([]v1beta1.ComponentDefinition, error) continue } if comp.Metadata == nil { - comp.Metadata = make(map[string]interface{}) + comp.Metadata = &v1beta1.ComponentDefinition_Metadata{} } - if comp.Model.Metadata == nil { - comp.Model.Metadata = make(map[string]interface{}) + if comp.Model.Metadata.AdditionalProperties == nil { + comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) } - comp.Model.Metadata["source_uri"] = pkg.ChartUrl + comp.Model.Metadata.AdditionalProperties["source_uri"] = pkg.ChartUrl comp.Model.Version = pkg.Version comp.Model.Name = pkg.Name - comp.Model.Category = v1beta1.Category{ + comp.Model.Category = v1beta1.CategoryDefinition{ Name: "", } comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) diff --git a/generators/github/package.go b/generators/github/package.go index 53411699..d99a1cff 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -4,7 +4,7 @@ import ( "bytes" "os" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/meshery/schemas/models/v1beta1" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" "github.com/layer5io/meshkit/utils/manifests" @@ -40,16 +40,16 @@ func (gp GitHubPackage) GenerateComponents() ([]v1beta1.ComponentDefinition, err continue } if comp.Metadata == nil { - comp.Metadata = make(map[string]interface{}) + comp.Metadata = &v1beta1.ComponentDefinition_Metadata{} } - if comp.Model.Metadata == nil { - comp.Model.Metadata = make(map[string]interface{}) + if comp.Model.Metadata.AdditionalProperties == nil { + comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) } - comp.Model.Metadata["source_uri"] = gp.SourceURL + comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL comp.Model.Version = gp.version comp.Model.Name = gp.Name - comp.Model.Category = v1beta1.Category{ + comp.Model.Category = v1beta1.CategoryDefinition{ Name: "", } comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) diff --git a/generators/models/interfaces.go b/generators/models/interfaces.go index 7d68c209..24a4e316 100644 --- a/generators/models/interfaces.go +++ b/generators/models/interfaces.go @@ -1,6 +1,7 @@ package models -import "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" +import "github.com/meshery/schemas/models/v1beta1" + // anything that can be validated is a Validator type Validator interface { diff --git a/go.mod b/go.mod index d413cafa..3b2f4238 100644 --- a/go.mod +++ b/go.mod @@ -263,3 +263,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) + +replace github.com/meshery/schemas => ../schemas diff --git a/go.sum b/go.sum index d2feebea..ef6fa25a 100644 --- a/go.sum +++ b/go.sum @@ -607,8 +607,6 @@ github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvls github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/meshery/kompose v1.0.1 h1:lg8B/pkLh6762jeFsQATD8UJZZwXZf/aviC3/dzw78A= github.com/meshery/kompose v1.0.1/go.mod h1:TWhWTEMbJBUzENf4JTEtBmZRFm/r8n0nS6v4/nSD2vA= -github.com/meshery/schemas v0.7.9 h1:7rA9RfRfbYRGONYMUbfgen2j+jsKgfjqHPuUYOhjwyA= -github.com/meshery/schemas v0.7.9/go.mod h1:ZsfoE5HvlqJvUvBlqS2rHoNQoDJ+eYuly5w5m+qIQSM= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go index 931ef10f..c7304439 100644 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -5,9 +5,10 @@ import ( "path/filepath" "strings" - "github.com/google/uuid" + "github.com/gofrs/uuid" "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/meshery/schemas/models/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/utils" "gorm.io/gorm/clause" @@ -16,9 +17,10 @@ import ( const RelationshipSchemaVersion = "relationships.meshery.io/v1alpha2" type RelationshipDefinition struct { - ID uuid.UUID `json:"id"` - v1beta1.VersionMeta `json:",inline" yaml:",inline"` - Kind string `json:"kind,omitempty" yaml:"kind"` + ID uuid.UUID `json:"id"` + SchemaVersion string `json:"schemaVersion,omitempty" yaml:"schemaVersion"` + Version string `json:"version,omitempty" yaml:"version"` + Kind string `json:"kind,omitempty" yaml:"kind"` // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. // Though, the column name and the json representation is "type". RelationshipType string `json:"type" yaml:"type" gorm:"type"` @@ -26,7 +28,7 @@ type RelationshipDefinition struct { EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` Metadata map[string]interface{} `json:"metadata" yaml:"metadata" gorm:"type:bytes;serializer:json"` ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:model_id"` - Model v1beta1.Model `json:"model" gorm:"foreignKey:ModelID;references:ID"` + Model v1beta1.ModelDefinition `json:"model" gorm:"foreignKey:ModelID;references:ID"` Selectors []map[string]interface{} `json:"selectors" yaml:"selectors" gorm:"type:bytes;serializer:json"` } @@ -39,7 +41,7 @@ func (r RelationshipDefinition) Type() entity.EntityType { } func (r *RelationshipDefinition) GenerateID() (uuid.UUID, error) { - return uuid.New(), nil + return uuid.NewV4() } func (r RelationshipDefinition) GetID() uuid.UUID { diff --git a/models/meshmodel/core/v1beta1/application_component.go b/models/meshmodel/core/v1beta1/application_component.go index b2b546b0..da95d6d0 100644 --- a/models/meshmodel/core/v1beta1/application_component.go +++ b/models/meshmodel/core/v1beta1/application_component.go @@ -1,68 +1,40 @@ package v1beta1 -import ( - "fmt" - "strings" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// To be removed when Design schema is refactored. - -// Component is the structure for the core Application Component -type Component struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec ComponentSpec `json:"spec,omitempty"` -} - -// ComponentSpec is the structure for the core Application Component Spec -type ComponentSpec struct { - Type string `json:"type"` - Version string `json:"version"` - APIVersion string `json:"apiVersion"` - Model string `json:"model"` - Settings map[string]interface{} `json:"settings"` - Parameters []ComponentParameter `json:"parameters"` -} - -// ComponentParameter is the structure for the core Application Component -// Paramater -type ComponentParameter struct { - Name string `json:"name"` - FieldPaths []string `json:"fieldPaths"` - Required *bool `json:"required,omitempty"` - Description *string `json:"description,omitempty"` -} - -const MesheryAnnotationPrefix = "design.meshmodel.io" - -func GetAPIVersionFromComponent(comp Component) string { - return comp.Annotations[MesheryAnnotationPrefix+".k8s.APIVersion"] -} - -func GetKindFromComponent(comp Component) string { - kind := strings.TrimPrefix(comp.Annotations[MesheryAnnotationPrefix+".k8s.Kind"], "/") - return kind -} - -func GetAnnotationsForWorkload(w ComponentDefinition) map[string]string { - res := map[string]string{} - - for key, val := range w.Metadata { - if v, ok := val.(string); ok { - res[strings.ReplaceAll(fmt.Sprintf("%s.%s", MesheryAnnotationPrefix, key), " ", "")] = v - } - } - sourceURI, ok := w.Model.Metadata["source_uri"].(string) - if ok { - res[fmt.Sprintf("%s.model.source_uri", MesheryAnnotationPrefix)] = sourceURI - } - res[fmt.Sprintf("%s.model.name", MesheryAnnotationPrefix)] = w.Model.Name - res[fmt.Sprintf("%s.k8s.APIVersion", MesheryAnnotationPrefix)] = w.Component.Version - res[fmt.Sprintf("%s.k8s.Kind", MesheryAnnotationPrefix)] = w.Component.Kind - res[fmt.Sprintf("%s.model.version", MesheryAnnotationPrefix)] = w.Model.Version - res[fmt.Sprintf("%s.model.category", MesheryAnnotationPrefix)] = w.Model.Category.Name - return res -} +// // ComponentParameter is the structure for the core Application Component +// // Paramater +// type ComponentParameter struct { +// Name string `json:"name"` +// FieldPaths []string `json:"fieldPaths"` +// Required *bool `json:"required,omitempty"` +// Description *string `json:"description,omitempty"` +// } + + +// func GetAPIVersionFromComponent(comp v1beta1.ComponentDefinition) string { +// return comp.Annotations[MesheryAnnotationPrefix+".k8s.APIVersion"] +// } + +// func GetKindFromComponent(comp v1beta1.ComponentDefinition) string { +// kind := strings.TrimPrefix(comp.Annotations[MesheryAnnotationPrefix+".k8s.Kind"], "/") +// return kind +// } + +// func GetAnnotationsForWorkload(w v1beta1.ComponentDefinition) map[string]string { +// res := map[string]string{} + +// for key, val := range w.Metadata { +// if v, ok := val.(string); ok { +// res[strings.ReplaceAll(fmt.Sprintf("%s.%s", MesheryAnnotationPrefix, key), " ", "")] = v +// } +// } +// sourceURI, ok := w.Model.Metadata["source_uri"].(string) +// if ok { +// res[fmt.Sprintf("%s.model.source_uri", MesheryAnnotationPrefix)] = sourceURI +// } +// res[fmt.Sprintf("%s.model.name", MesheryAnnotationPrefix)] = w.Model.Name +// res[fmt.Sprintf("%s.k8s.APIVersion", MesheryAnnotationPrefix)] = w.Component.Version +// res[fmt.Sprintf("%s.k8s.Kind", MesheryAnnotationPrefix)] = w.Component.Kind +// res[fmt.Sprintf("%s.model.version", MesheryAnnotationPrefix)] = w.Model.Version +// res[fmt.Sprintf("%s.model.category", MesheryAnnotationPrefix)] = w.Model.Category.Name +// return res +// } diff --git a/models/meshmodel/core/v1beta1/category.go b/models/meshmodel/core/v1beta1/category.go deleted file mode 100644 index 8d60a202..00000000 --- a/models/meshmodel/core/v1beta1/category.go +++ /dev/null @@ -1,83 +0,0 @@ -package v1beta1 - -import ( - "encoding/json" - "fmt" - "sync" - - "github.com/google/uuid" - "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/entity" - "gorm.io/gorm" -) - -var categoryCreationLock sync.Mutex //Each model will perform a check and if the category already doesn't exist, it will create a category. This lock will make sure that there are no race conditions. - -// swagger:response Category -type Category struct { - ID uuid.UUID `json:"-"` - Name string `json:"name" gorm:"name"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata" gorm:"type:bytes;serializer:json"` -} - -func (c Category) TableName() string { - return "category_dbs" -} - -// "Uncategorized" is assigned when Category is empty in the component definitions. -const DefaultCategory = "Uncategorized" - -func (cat Category) Type() entity.EntityType { - return entity.Category -} - -func (cat *Category) GenerateID() (uuid.UUID, error) { - categoryIdentifier := Category{ - Name: cat.Name, - } - byt, err := json.Marshal(categoryIdentifier) - if err != nil { - return uuid.UUID{}, err - } - catID := uuid.NewSHA1(uuid.UUID{}, byt) - return catID, nil -} - -func (cat Category) GetID() uuid.UUID { - return cat.ID -} - -func (cat *Category) GetEntityDetail() string { - return fmt.Sprintf("name: %s", cat.Name) -} - -func (cat *Category) Create(db *database.Handler, _ uuid.UUID) (uuid.UUID, error) { - if cat.Name == "" { - cat.Name = DefaultCategory - } - - catID, err := cat.GenerateID() - if err != nil { - return catID, err - } - var category Category - categoryCreationLock.Lock() - defer categoryCreationLock.Unlock() - err = db.First(&category, "id = ?", catID).Error - if err != nil && err != gorm.ErrRecordNotFound { - return uuid.UUID{}, err - } - if err == gorm.ErrRecordNotFound { //The category is already not present and needs to be inserted - cat.ID = catID - err = db.Create(&cat).Error - if err != nil { - return uuid.UUID{}, err - } - return cat.ID, nil - } - return category.ID, nil -} - -func (m *Category) UpdateStatus(db database.Handler, status entity.EntityStatus) error { - return nil -} diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go deleted file mode 100644 index 0722ec41..00000000 --- a/models/meshmodel/core/v1beta1/component.go +++ /dev/null @@ -1,109 +0,0 @@ -package v1beta1 - -import ( - "fmt" - "path/filepath" - - "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/utils" - "gorm.io/gorm/clause" - - "github.com/google/uuid" -) - -type VersionMeta struct { - SchemaVersion string `json:"schemaVersion,omitempty" yaml:"schemaVersion" gorm:"column:schema_version"` - Version string `json:"version,omitempty" yaml:"version" gorm:"column:version"` -} - -type TypeMeta struct { - Kind string `json:"kind,omitempty" yaml:"kind"` - Version string `json:"version,omitempty" yaml:"version"` -} - -type ComponentFormat string - -const ( - JSON ComponentFormat = "JSON" - YAML ComponentFormat = "YAML" - CUE ComponentFormat = "CUE" - - ComponentSchemaVersion = "components.meshery.io/v1beta1" -) - -// Contains information as extracted from the core underlying component eg: Pod's apiVersion, kind and schema -type ComponentEntity struct { - TypeMeta - Schema string `json:"schema,omitempty" yaml:"schema"` -} - -// swagger:response ComponentDefinition -type ComponentDefinition struct { - ID uuid.UUID `json:"id"` - VersionMeta `json:",inline" yaml:",inline"` - DisplayName string `json:"displayName" gorm:"column:display_name"` - Description string `json:"description" gorm:"column:description"` - Format ComponentFormat `json:"format" yaml:"format"` - ModelID uuid.UUID `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:model_id"` - Model Model `json:"model" gorm:"foreignKey:ModelID;references:ID"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata" gorm:"type:bytes;serializer:json"` - Component ComponentEntity `json:"component,omitempty" yaml:"component" gorm:"type:bytes;serializer:json"` -} - -func (c ComponentDefinition) TableName() string { - return "component_definition_dbs" -} - -func (c ComponentDefinition) Type() entity.EntityType { - return entity.ComponentDefinition -} - -func (c *ComponentDefinition) GenerateID() (uuid.UUID, error) { - return uuid.New(), nil -} - -func (c ComponentDefinition) GetID() uuid.UUID { - return c.ID -} - -func (c *ComponentDefinition) GetEntityDetail() string { - return fmt.Sprintf("type: %s, definition version: %s, name: %s, model: %s, version: %s", c.Type(), c.Version, c.DisplayName, c.Model.Name, c.Model.Version) -} - -func (c *ComponentDefinition) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error) { - c.ID, _ = c.GenerateID() - - isAnnotation, _ := c.Metadata["isAnnotation"].(bool) - - if c.Component.Schema == "" && !isAnnotation { //For components which has an empty schema and is not an annotation, return error - // return ErrEmptySchema() - return uuid.Nil, nil - } - - mid, err := c.Model.Create(db, hostID) - if err != nil { - return uuid.UUID{}, err - } - - if !utils.IsSchemaEmpty(c.Component.Schema) { - c.Metadata["hasInvalidSchema"] = true - } - - c.ModelID = mid - err = db.Omit(clause.Associations).Create(&c).Error - return c.ID, err -} - -func (m *ComponentDefinition) UpdateStatus(db *database.Handler, status entity.EntityStatus) error { - return nil -} - -func (c ComponentDefinition) WriteComponentDefinition(componentDirPath string) error { - if c.Component.Kind == "" { - return nil - } - componentPath := filepath.Join(componentDirPath, c.Component.Kind+".json") - err := utils.WriteJSONToFile[ComponentDefinition](componentPath, c) - return err -} diff --git a/models/meshmodel/core/v1beta1/host.go b/models/meshmodel/core/v1beta1/host.go index 2c64e03b..e3070d04 100644 --- a/models/meshmodel/core/v1beta1/host.go +++ b/models/meshmodel/core/v1beta1/host.go @@ -1,29 +1,14 @@ package v1beta1 import ( - "encoding/json" "fmt" - "sync" - "time" "github.com/google/uuid" - "github.com/layer5io/meshkit/database" + "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/kubernetes" - "gorm.io/gorm" + "github.com/meshery/schemas/models/v1beta1" ) -var hostCreationLock sync.Mutex //Each entity will perform a check and if the host already doesn't exist, it will create a host. This lock will make sure that there are no race conditions. - -type Host struct { - ID uuid.UUID `json:"-"` - Hostname string `json:"hostname"` - Port int `json:"port,omitempty"` - Metadata string `json:"metadata,omitempty"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` - IHost IHost `json:"-" gorm:"-"` -} - type MeshModelHostsWithEntitySummary struct { ID uuid.UUID `json:"id"` Hostname string `json:"hostname"` @@ -58,90 +43,45 @@ type HostFilter struct { Offset int } -func (h *Host) GenerateID() (uuid.UUID, error) { - byt, err := json.Marshal(h) - if err != nil { - return uuid.UUID{}, err - } - hID := uuid.NewSHA1(uuid.UUID{}, byt) - return hID, nil -} - -func (h *Host) Create(db *database.Handler) (uuid.UUID, error) { - - hID, err := h.GenerateID() - if err != nil { - return uuid.UUID{}, err - } - var host Host - hostCreationLock.Lock() - defer hostCreationLock.Unlock() - err = db.First(&host, "id = ?", hID).Error // check if the host already exists - if err != nil && err != gorm.ErrRecordNotFound { - return uuid.UUID{}, err - } - - // if not exists then create a new host and return the id - if err == gorm.ErrRecordNotFound { - h.ID = hID - err = db.Create(&h).Error - if err != nil { - return uuid.UUID{}, err - } - return h.ID, nil - } - - // else return the id of the existing host - return host.ID, nil -} - -func (h *Host) AfterFind(tx *gorm.DB) error { - switch h.Hostname { - case "artifacthub": - h.IHost = ArtifactHub{} - case "kubernetes": - h.IHost = Kubernetes{} - default: // do nothing if the host is not pre-unknown. Currently adapters fall into this case. - return nil - } - return nil -} - // Each host from where meshmodels can be generated needs to implement this interface // HandleDependents, contains host specific logic for provisioning required CRDs/operators for corresponding components. -type IHost interface { - HandleDependents(comp Component, kc *kubernetes.Client, isDeploy, performUpgrade bool) (string, error) - String() string -} type ArtifactHub struct{} -func (ah ArtifactHub) HandleDependents(comp Component, kc *kubernetes.Client, isDeploy, performUpgrade bool) (summary string, err error) { - source_uri := comp.Annotations[fmt.Sprintf("%s.model.source_uri", MesheryAnnotationPrefix)] +const MesheryAnnotationPrefix = "design.meshmodel.io" + +func (ah ArtifactHub) HandleDependents(comp v1beta1.ComponentDefinition, kc *kubernetes.Client, isDeploy, performUpgrade bool) (summary string, err error) { + annotations, err := utils.Cast[map[string]interface{}](comp.Metadata.AdditionalProperties["annotations"]) + if err != nil { + return summary, err + } + act := kubernetes.UNINSTALL if isDeploy { act = kubernetes.INSTALL } - if source_uri != "" { + source_uri, ok := annotations[fmt.Sprintf("%s.model.source_uri", MesheryAnnotationPrefix)] + + if ok && source_uri != "" { err = kc.ApplyHelmChart(kubernetes.ApplyHelmChartConfig{ - URL: source_uri, - Namespace: comp.Namespace, + URL: source_uri.(string), + Namespace: comp.Configuration["namespace"].(string), CreateNamespace: true, Action: act, UpgradeIfInstalled: performUpgrade, }) if err != nil { if !isDeploy { - summary = fmt.Sprintf("error undeploying dependent helm chart for %s, please proceed with manual uninstall or try again", comp.Name) + summary = fmt.Sprintf("error undeploying dependent helm chart for %s, please proceed with manual uninstall or try again", comp.DisplayName) } else { - summary = fmt.Sprintf("error deploying dependent helm chart for %s, please procced with manual install or try again", comp.Name) + summary = fmt.Sprintf("error deploying dependent helm chart for %s, please procced with manual install or try again", comp.DisplayName) } } else { if !isDeploy { - summary = fmt.Sprintf("Undeployed dependent helm chart for %s", comp.Name) + summary = fmt.Sprintf("Undeployed dependent helm chart for %s", comp.DisplayName) } else { - summary = fmt.Sprintf("Deployed dependent helm chart for %s", comp.Name) + summary = fmt.Sprintf("Deployed dependent helm chart for %s", comp.DisplayName) } } } @@ -154,7 +94,7 @@ func (ah ArtifactHub) String() string { type Kubernetes struct{} -func (k Kubernetes) HandleDependents(comp Component, kc *kubernetes.Client, isDeploy, performUpgrade bool) (summary string, err error) { +func (k Kubernetes) HandleDependents(_ v1beta1.ComponentDefinition, _ *kubernetes.Client, _, _ bool) (summary string, err error) { return summary, err } diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go deleted file mode 100644 index b56ee887..00000000 --- a/models/meshmodel/core/v1beta1/models.go +++ /dev/null @@ -1,154 +0,0 @@ -package v1beta1 - -import ( - "encoding/json" - "fmt" - "path/filepath" - "sync" - - "github.com/google/uuid" - "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/utils" - "gorm.io/gorm" - "gorm.io/gorm/clause" -) - -const ModelSchemaVersion = "models.meshery.io/v1beta1" - -var modelCreationLock sync.Mutex //Each component/relationship will perform a check and if the model already doesn't exist, it will create a model. This lock will make sure that there are no race conditions. - -type ModelEntity struct { - Version string `json:"version,omitempty" yaml:"version"` -} - -// swagger:response Model -type Model struct { - ID uuid.UUID `json:"id"` - VersionMeta `yaml:",inline"` - - Name string `json:"name" gorm:"modelName"` - DisplayName string `json:"displayName"` - Description string `json:"description" gorm:"description"` - Status entity.EntityStatus `json:"status" gorm:"status"` - RegistrantID uuid.UUID `json:"hostID" gorm:"column:host_id"` // make as a foreign refer to host's table - Registrant Host `json:"registrant" gorm:"foreignKey:RegistrantID;references:ID"` - CategoryID uuid.UUID `json:"-" gorm:"categoryID"` - Category Category `json:"category" gorm:"foreignKey:CategoryID;references:ID"` - SubCategory string `json:"subCategory" gorm:"subCategory"` - Metadata map[string]interface{} `json:"metadata" gorm:"type:bytes;serializer:json"` - Model ModelEntity `json:"model,omitempty" gorm:"model;type:bytes;serializer:json"` - Components []ComponentDefinition `json:"components" gorm:"-"` - Relationships interface{} `json:"relationships" gorm:"-"` -} - -func (m Model) TableName() string { - return "model_dbs" -} - -func (m Model) Type() entity.EntityType { - return entity.Model -} - -func (m *Model) GenerateID() (uuid.UUID, error) { - modelIdentifier := Model{ - Registrant: m.Registrant, - VersionMeta: m.VersionMeta, - Name: m.Name, - Model: ModelEntity{ - Version: m.Model.Version, - }, - } - byt, err := json.Marshal(modelIdentifier) - if err != nil { - return uuid.UUID{}, err - } - return uuid.NewSHA1(uuid.UUID{}, byt), nil -} - -func (m Model) GetID() uuid.UUID { - return m.ID -} - -func (m *Model) GetEntityDetail() string { - return fmt.Sprintf("type: %s, model: %s, definition version: %s, version: %s", m.Type(), m.Name, m.Version, m.Model.Version) -} - -func (m *Model) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error) { - modelID, err := m.GenerateID() - if err != nil { - return modelID, err - } - - var model Model - if m.Name == "" { - return uuid.UUID{}, fmt.Errorf("empty or invalid model name passed") - } - modelCreationLock.Lock() - defer modelCreationLock.Unlock() - err = db.First(&model, "id = ? and host_id = ?", modelID, hostID).Error - if err != nil && err != gorm.ErrRecordNotFound { - return uuid.UUID{}, err - } - if err == gorm.ErrRecordNotFound { //The model is already not present and needs to be inserted - id, err := m.Category.Create(db, hostID) - if err != nil { - return uuid.UUID{}, err - } - m.ID = modelID - m.CategoryID = id - m.RegistrantID = hostID - err = db.Omit(clause.Associations).Create(&m).Error - if err != nil { - return uuid.UUID{}, err - } - err = registerModel(db, hostID, modelID) - if err != nil { - return uuid.UUID{}, err - } - return m.ID, nil - } - return model.ID, nil -} -func registerModel(db *database.Handler, regID, modelID uuid.UUID) error { - var count int64 - err := db.Table("registries").Where("registrant_id=?", regID).Where("type = ?", "model").Where("entity = ?", modelID).Count(&count).Error - if err != nil && err != gorm.ErrRecordNotFound { - return err - } - if count == 0 { - err = db.Exec("INSERT INTO registries (registrant_id, entity,type) VALUES (?, ?, ?)", regID, modelID, "model").Error - if err != nil { - return err - } - } - return nil -} - -func (m *Model) UpdateStatus(db *database.Handler, status entity.EntityStatus) error { - err := db.Model(&Model{}).Where("id = ?", m.ID).Update("status", status).Error - if err != nil { - return entity.ErrUpdateEntityStatus(err, string(m.Type()), status) - } - return nil -} - -// WriteModelDefinition writes out the model to the given `modelDefPath` in the `outputType` format. -// `outputType` can be `yaml` or `json`. -// Usage: model.WriteModelDefinition("./modelName/model.yaml", "yaml") -func (c Model) WriteModelDefinition(modelDefPath string, outputType string) error { - err := utils.CreateDirectory(filepath.Dir(modelDefPath)) - if err != nil { - return err - } - if outputType == "json" { - err = utils.WriteJSONToFile[Model](modelDefPath, c) - } - if outputType == "yaml" { - err = utils.WriteYamlToFile[Model](modelDefPath, c) - } - if err != nil { - return err - } - return nil -} diff --git a/models/meshmodel/core/v1beta1/policy.go b/models/meshmodel/core/v1beta1/policy.go index a4f68015..03c4e417 100644 --- a/models/meshmodel/core/v1beta1/policy.go +++ b/models/meshmodel/core/v1beta1/policy.go @@ -5,9 +5,10 @@ import ( "path/filepath" "time" - "github.com/google/uuid" + "github.com/gofrs/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/utils" + "github.com/meshery/schemas/models/v1beta1" "gorm.io/gorm/clause" "github.com/layer5io/meshkit/models/meshmodel/entity" @@ -15,14 +16,15 @@ import ( // swagger:response PolicyDefinition type PolicyDefinition struct { - ID uuid.UUID `json:"-"` - TypeMeta - ModelID uuid.UUID `json:"-" gorm:"column:modelID"` - Model Model `json:"model"` - SubType string `json:"subType" yaml:"subType"` - Expression map[string]interface{} `json:"expression" yaml:"expression" gorm:"type:bytes;serializer:json"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` + ID uuid.UUID `json:"-"` + Kind string `json:"kind,omitempty" yaml:"kind"` + Version string `json:"version,omitempty" yaml:"version"` + ModelID uuid.UUID `json:"-" gorm:"column:modelID"` + Model v1beta1.ModelDefinition `json:"model"` + SubType string `json:"subType" yaml:"subType"` + Expression map[string]interface{} `json:"expression" yaml:"expression" gorm:"type:bytes;serializer:json"` + CreatedAt time.Time `json:"-"` + UpdatedAt time.Time `json:"-"` } func (p PolicyDefinition) GetID() uuid.UUID { @@ -30,7 +32,7 @@ func (p PolicyDefinition) GetID() uuid.UUID { } func (p *PolicyDefinition) GenerateID() (uuid.UUID, error) { - return uuid.New(), nil + return uuid.NewV4() } func (p PolicyDefinition) Type() entity.EntityType { diff --git a/models/meshmodel/entity/types.go b/models/meshmodel/entity/types.go index b7d0a96d..37f55d46 100644 --- a/models/meshmodel/entity/types.go +++ b/models/meshmodel/entity/types.go @@ -1,7 +1,7 @@ package entity import ( - "github.com/google/uuid" + "github.com/gofrs/uuid" "github.com/layer5io/meshkit/database" ) diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index fbd0558a..cd3b18f8 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -5,10 +5,11 @@ import ( "strings" "time" - "github.com/google/uuid" + "github.com/gofrs/uuid" "github.com/layer5io/meshkit/database" + "github.com/meshery/schemas/models/v1beta1" + model "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" "golang.org/x/text/cases" "golang.org/x/text/language" @@ -23,9 +24,9 @@ import ( // 2. Entity type // 3. Entity type MeshModelRegistrantData struct { - Host v1beta1.Host `json:"host"` - EntityType entity.EntityType `json:"entityType"` - Entity []byte `json:"entity"` //This will be type converted to appropriate entity on server based on passed entity type + Connection v1beta1.Connection `json:"connection"` + EntityType entity.EntityType `json:"entityType"` + Entity []byte `json:"entity"` //This will be type converted to appropriate entity on server based on passed entity type } type Registry struct { ID uuid.UUID @@ -52,12 +53,12 @@ func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { } err := rm.db.AutoMigrate( &Registry{}, - &v1beta1.Host{}, + &v1beta1.Connection{}, &v1beta1.ComponentDefinition{}, &v1alpha2.RelationshipDefinition{}, - &v1beta1.PolicyDefinition{}, - &v1beta1.Model{}, - &v1beta1.Category{}, + // &v1beta1.PolicyDefinition{}, + &v1beta1.ModelDefinition{}, + &v1beta1.CategoryDefinition{}, ) if err != nil { return nil, err @@ -67,14 +68,14 @@ func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { func (rm *RegistryManager) Cleanup() { _ = rm.db.Migrator().DropTable( &Registry{}, - &v1beta1.Host{}, + &v1beta1.Connection{}, &v1beta1.ComponentDefinition{}, - &v1beta1.Model{}, - &v1beta1.Category{}, + &v1beta1.ModelDefinition{}, + &v1beta1.CategoryDefinition{}, &v1alpha2.RelationshipDefinition{}, ) } -func (rm *RegistryManager) RegisterEntity(h v1beta1.Host, en entity.Entity) (bool, bool, error) { +func (rm *RegistryManager) RegisterEntity(h v1beta1.Connection, en entity.Entity) (bool, bool, error) { registrantID, err := h.Create(rm.db) if err != nil { return true, false, err @@ -84,8 +85,9 @@ func (rm *RegistryManager) RegisterEntity(h v1beta1.Host, en entity.Entity) (boo if err != nil { return false, true, err } + id, _ := uuid.NewV4() entry := Registry{ - ID: uuid.New(), + ID: id, RegistrantID: registrantID, Entity: entityID, Type: en.Type(), @@ -103,13 +105,13 @@ func (rm *RegistryManager) RegisterEntity(h v1beta1.Host, en entity.Entity) (boo // By default during models generation ignore is set to false func (rm *RegistryManager) UpdateEntityStatus(ID string, status string, entityType string) error { // Convert string UUID to google UUID - entityID, err := uuid.Parse(ID) + entityID, err := uuid.FromString(ID) if err != nil { return err } switch entityType { case "models": - model := v1beta1.Model{ID: entityID} + model := v1beta1.ModelDefinition{Id: entityID} err := model.UpdateStatus(rm.db, entity.EntityStatus(status)) if err != nil { return err @@ -120,18 +122,18 @@ func (rm *RegistryManager) UpdateEntityStatus(ID string, status string, entityTy } } -func (rm *RegistryManager) GetRegistrant(e entity.Entity) v1beta1.Host { +func (rm *RegistryManager) GetRegistrant(e entity.Entity) v1beta1.Connection { eID := e.GetID() var reg Registry _ = rm.db.Where("entity = ?", eID).Find(®).Error - var h v1beta1.Host + var h v1beta1.Connection _ = rm.db.Where("id = ?", reg.RegistrantID).Find(&h).Error return h } // to be removed -func (rm *RegistryManager) GetRegistrants(f *v1beta1.HostFilter) ([]v1beta1.MeshModelHostsWithEntitySummary, int64, error) { - var result []v1beta1.MesheryHostSummaryDB +func (rm *RegistryManager) GetRegistrants(f *model.HostFilter) ([]model.MeshModelHostsWithEntitySummary, int64, error) { + var result []model.MesheryHostSummaryDB var totalcount int64 db := rm.db @@ -170,14 +172,14 @@ func (rm *RegistryManager) GetRegistrants(f *v1beta1.HostFilter) ([]v1beta1.Mesh return nil, 0, err } - var response []v1beta1.MeshModelHostsWithEntitySummary + var response []model.MeshModelHostsWithEntitySummary for _, r := range result { - res := v1beta1.MeshModelHostsWithEntitySummary{ + res := model.MeshModelHostsWithEntitySummary{ ID: r.HostID, Hostname: HostnameToPascalCase(r.Hostname), Port: r.Port, - Summary: v1beta1.EntitySummary{ + Summary: model.EntitySummary{ Models: r.Models, Components: r.Components, Relationships: r.Relationships, diff --git a/models/meshmodel/registry/v1beta1/category_filter.go b/models/meshmodel/registry/v1beta1/category_filter.go index b992e8d1..d5258d6b 100644 --- a/models/meshmodel/registry/v1beta1/category_filter.go +++ b/models/meshmodel/registry/v1beta1/category_filter.go @@ -2,9 +2,8 @@ package v1beta1 import ( "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/models/meshmodel/registry" + "github.com/meshery/schemas/models/v1beta1" "gorm.io/gorm/clause" ) @@ -36,7 +35,7 @@ func (cf *CategoryFilter) GetById(db *database.Handler) (entity.Entity, error) { } func (cf *CategoryFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { - var catdb []v1beta1.Category + var catdb []v1beta1.CategoryDefinition var cat []entity.Entity finder := db.Model(&catdb).Debug() diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index c6996cda..1d8c3a2a 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -2,9 +2,8 @@ package v1beta1 import ( "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/models/meshmodel/registry" + "github.com/meshery/schemas/models/v1beta1" "gorm.io/gorm/clause" ) @@ -28,9 +27,9 @@ type ComponentFilter struct { type componentDefinitionWithModel struct { ComponentDefinitionDB v1beta1.ComponentDefinition `gorm:"embedded"` - ModelDB v1beta1.Model `gorm:"embedded"` - CategoryDB v1beta1.Category `gorm:"embedded"` - HostsDB v1beta1.Host `gorm:"embedded"` + ModelDB v1beta1.ModelDefinition `gorm:"embedded"` + CategoryDB v1beta1.CategoryDefinition `gorm:"embedded"` + ConnectionDB v1beta1.Connection `gorm:"embedded"` } func (cf *ComponentFilter) GetById(db *database.Handler) (entity.Entity, error) { @@ -144,7 +143,7 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti cm.ComponentDefinitionDB.Component.Schema = "" } - reg := cm.HostsDB + reg := cm.ConnectionDB cd := cm.ComponentDefinitionDB cd.Model = cm.ModelDB cd.Model.Category = cm.CategoryDB diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index d5d3cce2..87c780de 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -3,9 +3,10 @@ package v1beta1 import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/models/meshmodel/registry" + "github.com/meshery/schemas/models/v1beta1" + "gorm.io/gorm/clause" ) @@ -37,7 +38,7 @@ func (mf *ModelFilter) Create(m map[string]interface{}) { mf.Name = m["name"].(string) } -func countUniqueModels(models []v1beta1.Model) int { +func countUniqueModels(models []v1beta1.ModelDefinition) int { set := make(map[string]struct{}) for _, model := range models { key := model.Name + "@" + model.Model.Version @@ -85,9 +86,9 @@ func (mf *ModelFilter) GetById(db *database.Handler) (entity.Entity, error) { } func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { - var modelWithCategories []v1beta1.Model + var modelWithCategories []v1beta1.ModelDefinition - finder := db.Model(&v1beta1.Model{}).Preload("Category").Preload("Registrant"). + finder := db.Model(&v1beta1.ModelDefinition{}).Preload("Category").Preload("Registrant"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id"). Joins("JOIN registries ON registries.entity = model_dbs.id"). Joins("JOIN hosts ON hosts.id = registries.registrant_id") @@ -188,11 +189,11 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e var relationships []v1alpha2.RelationshipDefinition finder := db.Model(&v1alpha2.RelationshipDefinition{}). Select("relationship_definition_dbs.*"). - Where("relationship_definition_dbs.model_id = ?", _modelDB.ID) + Where("relationship_definition_dbs.model_id = ?", _modelDB.Id) if err := finder.Scan(&relationships).Error; err != nil { return nil, 0, 0, err } - _modelDB.Relationships = relationships + // _modelDB.Relationships = relationships FIX THIS } defs = append(defs, &_modelDB) } diff --git a/utils/component/generator.go b/utils/component/generator.go index 25949854..975b0279 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -6,7 +6,7 @@ import ( "fmt" "cuelang.org/go/cue" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/meshery/schemas/models/v1beta1" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/manifests" ) @@ -50,7 +50,7 @@ func Generate(crd string) (v1beta1.ComponentDefinition, error) { component := v1beta1.ComponentDefinition{} component.SchemaVersion = v1beta1.ComponentSchemaVersion - component.Metadata = make(map[string]interface{}) + component.Metadata = &v1beta1.ComponentDefinition_Metadata{} crdCue, err := utils.YamlToCue(crd) if err != nil { return component, err @@ -76,11 +76,14 @@ func Generate(crd string) (v1beta1.ComponentDefinition, error) { return component, err } // return component, err Ignore error if scope isn't found + if component.Metadata.AdditionalProperties == nil { + component.Metadata.AdditionalProperties = make(map[string]interface{}) + } scope, _ := extractCueValueFromPath(crdCue, DefaultPathConfig.ScopePath) if scope == "Cluster" { - component.Metadata["isNamespaced"] = false + component.Metadata.AdditionalProperties["isNamespaced"] = false } else if scope == "Namespaced" { - component.Metadata["isNamespaced"] = true + component.Metadata.AdditionalProperties["isNamespaced"] = true } component.Component.Kind = name if group != "" { diff --git a/utils/component/generator_test.go b/utils/component/generator_test.go index ed26c8d6..2081f883 100644 --- a/utils/component/generator_test.go +++ b/utils/component/generator_test.go @@ -3,7 +3,7 @@ package component import ( "testing" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/meshery/schemas/models/v1beta1" "github.com/layer5io/meshkit/utils/manifests" ) diff --git a/validator/validate_test.go b/validator/validate_test.go index 5db1ec85..a0741736 100644 --- a/validator/validate_test.go +++ b/validator/validate_test.go @@ -5,8 +5,8 @@ import ( "testing" "github.com/layer5io/meshkit/models/catalog/v1alpha1" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/meshery/schemas/models/v1alpha2" + "github.com/meshery/schemas/models/v1beta1" ) type ValidationCases struct { @@ -42,15 +42,15 @@ func TestValidator(t *testing.T) { }, { Path: "models", - Resource: v1beta1.Model{ - VersionMeta: v1beta1.VersionMeta{ - SchemaVersion: "v1beta1", - Version: "1.0.0", - }, - Category: v1beta1.Category{ + Resource: v1beta1.ModelDefinition{ + + SchemaVersion: "v1beta1", + Version: "1.0.0", + + Category: v1beta1.CategoryDefinition{ Name: "test", }, - Model: v1beta1.ModelEntity{ + Model: v1beta1.Model{ Version: "1.0.0", }, Status: "", From 8a572f118145e47a9364bc105d08302c3db3f10d Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 30 Jul 2024 17:27:47 +0530 Subject: [PATCH 018/112] udpate Signed-off-by: MUzairS15 --- cmd/syncmodutil/internal/modsync/sync.go | 2 +- generators/artifacthub/const.go | 5 ++++ generators/artifacthub/package.go | 6 ++--- generators/github/const.go | 5 ++++ generators/github/package.go | 5 +--- generators/models/interfaces.go | 3 +-- logger/logger.go | 4 +-- .../core/v1beta1/application_component.go | 1 - models/meshmodel/core/v1beta1/error.go | 11 ++++++++ models/meshmodel/core/v1beta1/host.go | 27 ++++++++++++++----- models/meshmodel/registry/registry.go | 4 +-- utils/catalog/package.go | 2 +- utils/component/generator.go | 4 +-- utils/component/generator_test.go | 2 +- utils/cue.go | 2 +- validator/validate.go | 2 +- 16 files changed, 56 insertions(+), 29 deletions(-) create mode 100644 generators/artifacthub/const.go create mode 100644 generators/github/const.go create mode 100644 models/meshmodel/core/v1beta1/error.go diff --git a/cmd/syncmodutil/internal/modsync/sync.go b/cmd/syncmodutil/internal/modsync/sync.go index 21c0864a..2f4c62df 100644 --- a/cmd/syncmodutil/internal/modsync/sync.go +++ b/cmd/syncmodutil/internal/modsync/sync.go @@ -110,7 +110,7 @@ func getRequiredVersionsFromString(s string) (p []Package) { if pkg == "" { continue } - + pkgName, pkgVersion := getPackageAndVersionFromPackageVersion(pkg) if strings.HasPrefix(pkgName, "//") { //Has been commented out continue diff --git a/generators/artifacthub/const.go b/generators/artifacthub/const.go new file mode 100644 index 00000000..db4c012f --- /dev/null +++ b/generators/artifacthub/const.go @@ -0,0 +1,5 @@ +package artifacthub + +const ( + ArtifactHub = "artifacthub" +) diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index 15c2332f..be3b8466 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -7,10 +7,10 @@ import ( "strings" "time" - "github.com/meshery/schemas/models/v1beta1" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" "github.com/layer5io/meshkit/utils/manifests" + "github.com/meshery/schemas/models/v1beta1" "gopkg.in/yaml.v2" ) @@ -52,9 +52,7 @@ func (pkg AhPackage) GenerateComponents() ([]v1beta1.ComponentDefinition, error) if err != nil { continue } - if comp.Metadata == nil { - comp.Metadata = &v1beta1.ComponentDefinition_Metadata{} - } + if comp.Model.Metadata.AdditionalProperties == nil { comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) } diff --git a/generators/github/const.go b/generators/github/const.go new file mode 100644 index 00000000..a572bb0b --- /dev/null +++ b/generators/github/const.go @@ -0,0 +1,5 @@ +package github + +const ( + GitHub = "github" +) diff --git a/generators/github/package.go b/generators/github/package.go index d99a1cff..93ef1577 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -4,10 +4,10 @@ import ( "bytes" "os" - "github.com/meshery/schemas/models/v1beta1" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" "github.com/layer5io/meshkit/utils/manifests" + "github.com/meshery/schemas/models/v1beta1" ) type GitHubPackage struct { @@ -39,9 +39,6 @@ func (gp GitHubPackage) GenerateComponents() ([]v1beta1.ComponentDefinition, err if err != nil { continue } - if comp.Metadata == nil { - comp.Metadata = &v1beta1.ComponentDefinition_Metadata{} - } if comp.Model.Metadata.AdditionalProperties == nil { comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) } diff --git a/generators/models/interfaces.go b/generators/models/interfaces.go index 24a4e316..507bebc8 100644 --- a/generators/models/interfaces.go +++ b/generators/models/interfaces.go @@ -1,7 +1,6 @@ package models -import "github.com/meshery/schemas/models/v1beta1" - +import "github.com/meshery/schemas/models/v1beta1" // anything that can be validated is a Validator type Validator interface { diff --git a/logger/logger.go b/logger/logger.go index 44ea184d..7f8ff1b8 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -1,9 +1,9 @@ package logger import ( + "fmt" "io" "os" - "fmt" "time" "github.com/go-logr/logr" @@ -132,4 +132,4 @@ func (l *Logger) Warnf(format string, args ...interface{}) { func (l *Logger) Debugf(format string, args ...interface{}) { l.handler.Log(logrus.DebugLevel, fmt.Sprintf(format, args...)) -} \ No newline at end of file +} diff --git a/models/meshmodel/core/v1beta1/application_component.go b/models/meshmodel/core/v1beta1/application_component.go index da95d6d0..e2e4209b 100644 --- a/models/meshmodel/core/v1beta1/application_component.go +++ b/models/meshmodel/core/v1beta1/application_component.go @@ -9,7 +9,6 @@ package v1beta1 // Description *string `json:"description,omitempty"` // } - // func GetAPIVersionFromComponent(comp v1beta1.ComponentDefinition) string { // return comp.Annotations[MesheryAnnotationPrefix+".k8s.APIVersion"] // } diff --git a/models/meshmodel/core/v1beta1/error.go b/models/meshmodel/core/v1beta1/error.go new file mode 100644 index 00000000..32e645b6 --- /dev/null +++ b/models/meshmodel/core/v1beta1/error.go @@ -0,0 +1,11 @@ +package v1beta1 + +import "github.com/layer5io/meshkit/errors" + +const ( + ErrUnknownKindCode = "" +) + +func ErrUnknownKind(err error) error { + return errors.New(ErrUnknownKindCode, errors.Alert, []string{"unsupported connection kind detected"}, []string{err.Error()}, []string{"The component's registrant is not supported by the version of server you are running"}, []string{"Try upgrading to latest available version"}) +} diff --git a/models/meshmodel/core/v1beta1/host.go b/models/meshmodel/core/v1beta1/host.go index e3070d04..60843b24 100644 --- a/models/meshmodel/core/v1beta1/host.go +++ b/models/meshmodel/core/v1beta1/host.go @@ -43,7 +43,22 @@ type HostFilter struct { Offset int } -// Each host from where meshmodels can be generated needs to implement this interface +type DependencyHandler interface { + HandleDependents(comp v1beta1.ComponentDefinition, kc *kubernetes.Client, isDeploy, performUpgrade bool) (string, error) + String() string +} + +func NewDependencyHandler(connectionKind string) (DependencyHandler, error) { + switch connectionKind { + case "kubernetes": + return Kubernetes{}, nil + case "artifacthub": + return ArtifactHub{}, nil + } + return nil, ErrUnknownKind(fmt.Errorf("unknown kind %s", connectionKind)) +} + +// Each connection from where meshmodels can be generated needs to implement this interface // HandleDependents, contains host specific logic for provisioning required CRDs/operators for corresponding components. type ArtifactHub struct{} @@ -51,21 +66,19 @@ type ArtifactHub struct{} const MesheryAnnotationPrefix = "design.meshmodel.io" func (ah ArtifactHub) HandleDependents(comp v1beta1.ComponentDefinition, kc *kubernetes.Client, isDeploy, performUpgrade bool) (summary string, err error) { - annotations, err := utils.Cast[map[string]interface{}](comp.Metadata.AdditionalProperties["annotations"]) + sourceURI, err := utils.Cast[string](comp.Metadata.AdditionalProperties["source_uri"]) // should be part of registrant data(?) if err != nil { return summary, err } - + act := kubernetes.UNINSTALL if isDeploy { act = kubernetes.INSTALL } - source_uri, ok := annotations[fmt.Sprintf("%s.model.source_uri", MesheryAnnotationPrefix)] - - if ok && source_uri != "" { + if sourceURI != "" { err = kc.ApplyHelmChart(kubernetes.ApplyHelmChartConfig{ - URL: source_uri.(string), + URL: sourceURI, Namespace: comp.Configuration["namespace"].(string), CreateNamespace: true, Action: act, diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index cd3b18f8..c7f13311 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -7,10 +7,10 @@ import ( "github.com/gofrs/uuid" "github.com/layer5io/meshkit/database" - "github.com/meshery/schemas/models/v1beta1" - model "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" + model "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/meshery/schemas/models/v1beta1" "golang.org/x/text/cases" "golang.org/x/text/language" "gorm.io/gorm/clause" diff --git a/utils/catalog/package.go b/utils/catalog/package.go index c0860fc4..3eb4abaf 100644 --- a/utils/catalog/package.go +++ b/utils/catalog/package.go @@ -2,9 +2,9 @@ package catalog import ( "fmt" + "github.com/layer5io/meshkit/models/catalog/v1alpha1" "regexp" "strings" - "github.com/layer5io/meshkit/models/catalog/v1alpha1" ) func BuildArtifactHubPkg(name, downloadURL, user, version, createdAt string, catalogData *v1alpha1.CatalogData) *ArtifactHubMetadata { diff --git a/utils/component/generator.go b/utils/component/generator.go index 975b0279..535583f2 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -6,9 +6,9 @@ import ( "fmt" "cuelang.org/go/cue" - "github.com/meshery/schemas/models/v1beta1" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/manifests" + "github.com/meshery/schemas/models/v1beta1" ) const ComponentMetaNameKey = "name" @@ -50,7 +50,7 @@ func Generate(crd string) (v1beta1.ComponentDefinition, error) { component := v1beta1.ComponentDefinition{} component.SchemaVersion = v1beta1.ComponentSchemaVersion - component.Metadata = &v1beta1.ComponentDefinition_Metadata{} + component.Metadata = v1beta1.ComponentDefinition_Metadata{} crdCue, err := utils.YamlToCue(crd) if err != nil { return component, err diff --git a/utils/component/generator_test.go b/utils/component/generator_test.go index 2081f883..b0759fc9 100644 --- a/utils/component/generator_test.go +++ b/utils/component/generator_test.go @@ -3,8 +3,8 @@ package component import ( "testing" - "github.com/meshery/schemas/models/v1beta1" "github.com/layer5io/meshkit/utils/manifests" + "github.com/meshery/schemas/models/v1beta1" ) var istioCrd = ` diff --git a/utils/cue.go b/utils/cue.go index 82254e86..c7854676 100644 --- a/utils/cue.go +++ b/utils/cue.go @@ -30,7 +30,7 @@ func Validate(schema cue.Value, value cue.Value) (bool, []errors.Error) { } return true }, nil) - + if len(errs) != 0 { return false, errs } diff --git a/validator/validate.go b/validator/validate.go index e1841908..a748d4f8 100644 --- a/validator/validate.go +++ b/validator/validate.go @@ -29,7 +29,7 @@ func loadSchema() error { if isSchemaLoaded { return nil } - + file, err := schemas.Schemas.Open("schemas/openapi.yml") if err != nil { return utils.ErrReadFile(err, "schemas/openapi.yml") From 2e4f2af886feb89a73470b9b469893cb5c899d19 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 31 Jul 2024 02:30:40 +0530 Subject: [PATCH 019/112] update registry logic Signed-off-by: MUzairS15 --- generators/artifacthub/package.go | 9 +-- generators/github/package.go | 9 +-- generators/models/interfaces.go | 2 +- .../core/policies/rego_policy_relationship.go | 1 + .../meshmodel/core/v1alpha2/relationship.go | 4 +- .../core/v1beta1/application_component.go | 6 +- models/meshmodel/core/v1beta1/host.go | 28 ++++---- models/meshmodel/core/v1beta1/policy.go | 2 +- models/meshmodel/registry/registry.go | 66 ++++++++++--------- .../registry/v1beta1/category_filter.go | 4 +- .../registry/v1beta1/component_filter.go | 32 +++++---- .../registry/v1beta1/model_filter.go | 11 ++-- utils/component/generator.go | 9 +-- utils/component/generator_test.go | 2 +- utils/kubernetes/apply-manifest.go | 2 + validator/validate_test.go | 10 +-- 16 files changed, 107 insertions(+), 90 deletions(-) diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index be3b8466..02d5c68d 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -10,7 +10,8 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" "github.com/layer5io/meshkit/utils/manifests" - "github.com/meshery/schemas/models/v1beta1" + "github.com/meshery/schemas/models/v1beta1/model" + "github.com/meshery/schemas/models/v1beta1/category" "gopkg.in/yaml.v2" ) @@ -36,8 +37,8 @@ func (pkg AhPackage) GetVersion() string { return pkg.Version } -func (pkg AhPackage) GenerateComponents() ([]v1beta1.ComponentDefinition, error) { - components := make([]v1beta1.ComponentDefinition, 0) +func (pkg AhPackage) GenerateComponents() ([]model.ComponentDefinition, error) { + components := make([]model.ComponentDefinition, 0) // TODO: Move this to the configuration if pkg.ChartUrl == "" { @@ -59,7 +60,7 @@ func (pkg AhPackage) GenerateComponents() ([]v1beta1.ComponentDefinition, error) comp.Model.Metadata.AdditionalProperties["source_uri"] = pkg.ChartUrl comp.Model.Version = pkg.Version comp.Model.Name = pkg.Name - comp.Model.Category = v1beta1.CategoryDefinition{ + comp.Model.Category = category.CategoryDefinition{ Name: "", } comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) diff --git a/generators/github/package.go b/generators/github/package.go index 93ef1577..42bc4e18 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -7,7 +7,8 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" "github.com/layer5io/meshkit/utils/manifests" - "github.com/meshery/schemas/models/v1beta1" + "github.com/meshery/schemas/models/v1beta1/category" + "github.com/meshery/schemas/models/v1beta1/model" ) type GitHubPackage struct { @@ -23,8 +24,8 @@ func (gp GitHubPackage) GetVersion() string { return gp.version } -func (gp GitHubPackage) GenerateComponents() ([]v1beta1.ComponentDefinition, error) { - components := make([]v1beta1.ComponentDefinition, 0) +func (gp GitHubPackage) GenerateComponents() ([]model.ComponentDefinition, error) { + components := make([]model.ComponentDefinition, 0) data, err := os.ReadFile(gp.filePath) if err != nil { @@ -46,7 +47,7 @@ func (gp GitHubPackage) GenerateComponents() ([]v1beta1.ComponentDefinition, err comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL comp.Model.Version = gp.version comp.Model.Name = gp.Name - comp.Model.Category = v1beta1.CategoryDefinition{ + comp.Model.Category = category.CategoryDefinition{ Name: "", } comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) diff --git a/generators/models/interfaces.go b/generators/models/interfaces.go index 507bebc8..00987db5 100644 --- a/generators/models/interfaces.go +++ b/generators/models/interfaces.go @@ -1,6 +1,6 @@ package models -import "github.com/meshery/schemas/models/v1beta1" +import v1beta1 "github.com/meshery/schemas/models/v1beta1/model" // anything that can be validated is a Validator type Validator interface { diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index 423116d3..0e258e7f 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -28,6 +28,7 @@ func NewRegoInstance(policyDir string, regManager *registry.RegistryManager) (*R ctx := context.Background() registeredRelationships, _, _, err := regManager.GetEntities(&v1alpha2.RelationshipFilter{}) if err != nil { + fmt.Println(err, ";;s;d;sd") return nil, err } diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go index c7304439..fc823bee 100644 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -7,7 +7,7 @@ import ( "github.com/gofrs/uuid" "github.com/layer5io/meshkit/database" - "github.com/meshery/schemas/models/v1beta1" + v1beta1 "github.com/meshery/schemas/models/v1beta1/model" "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/utils" @@ -28,7 +28,7 @@ type RelationshipDefinition struct { EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` Metadata map[string]interface{} `json:"metadata" yaml:"metadata" gorm:"type:bytes;serializer:json"` ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:model_id"` - Model v1beta1.ModelDefinition `json:"model" gorm:"foreignKey:ModelID;references:ID"` + Model v1beta1.ModelDefinition `json:"model" gorm:"foreignKey:ModelID;references:Id"` Selectors []map[string]interface{} `json:"selectors" yaml:"selectors" gorm:"type:bytes;serializer:json"` } diff --git a/models/meshmodel/core/v1beta1/application_component.go b/models/meshmodel/core/v1beta1/application_component.go index e2e4209b..1e96a0f7 100644 --- a/models/meshmodel/core/v1beta1/application_component.go +++ b/models/meshmodel/core/v1beta1/application_component.go @@ -9,16 +9,16 @@ package v1beta1 // Description *string `json:"description,omitempty"` // } -// func GetAPIVersionFromComponent(comp v1beta1.ComponentDefinition) string { +// func GetAPIVersionFromComponent(comp model.ComponentDefinition) string { // return comp.Annotations[MesheryAnnotationPrefix+".k8s.APIVersion"] // } -// func GetKindFromComponent(comp v1beta1.ComponentDefinition) string { +// func GetKindFromComponent(comp model.ComponentDefinition) string { // kind := strings.TrimPrefix(comp.Annotations[MesheryAnnotationPrefix+".k8s.Kind"], "/") // return kind // } -// func GetAnnotationsForWorkload(w v1beta1.ComponentDefinition) map[string]string { +// func GetAnnotationsForWorkload(w model.ComponentDefinition) map[string]string { // res := map[string]string{} // for key, val := range w.Metadata { diff --git a/models/meshmodel/core/v1beta1/host.go b/models/meshmodel/core/v1beta1/host.go index 60843b24..52acf1bb 100644 --- a/models/meshmodel/core/v1beta1/host.go +++ b/models/meshmodel/core/v1beta1/host.go @@ -3,17 +3,15 @@ package v1beta1 import ( "fmt" - "github.com/google/uuid" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/kubernetes" - "github.com/meshery/schemas/models/v1beta1" + "github.com/meshery/schemas/models/v1beta1/connection" + "github.com/meshery/schemas/models/v1beta1/model" ) type MeshModelHostsWithEntitySummary struct { - ID uuid.UUID `json:"id"` - Hostname string `json:"hostname"` - Port int `json:"port"` - Summary EntitySummary `json:"summary"` + connection.Connection + Summary EntitySummary `json:"summary"` } type EntitySummary struct { Models int64 `json:"models"` @@ -22,13 +20,11 @@ type EntitySummary struct { Policies int64 `json:"policies"` } type MesheryHostSummaryDB struct { - HostID uuid.UUID `json:"-" gorm:"id"` - Hostname string `json:"-" gorm:"hostname"` - Port int `json:"-" gorm:"port"` - Models int64 `json:"-" gorm:"models"` - Components int64 `json:"-" gorm:"components"` - Relationships int64 `json:"-" gorm:"relationships"` - Policies int64 `json:"-" gorm:"policies"` + connection.Connection + Models int64 `json:"-" gorm:"models"` + Components int64 `json:"-" gorm:"components"` + Relationships int64 `json:"-" gorm:"relationships"` + Policies int64 `json:"-" gorm:"policies"` } type HostFilter struct { @@ -44,7 +40,7 @@ type HostFilter struct { } type DependencyHandler interface { - HandleDependents(comp v1beta1.ComponentDefinition, kc *kubernetes.Client, isDeploy, performUpgrade bool) (string, error) + HandleDependents(comp model.ComponentDefinition, kc *kubernetes.Client, isDeploy, performUpgrade bool) (string, error) String() string } @@ -65,7 +61,7 @@ type ArtifactHub struct{} const MesheryAnnotationPrefix = "design.meshmodel.io" -func (ah ArtifactHub) HandleDependents(comp v1beta1.ComponentDefinition, kc *kubernetes.Client, isDeploy, performUpgrade bool) (summary string, err error) { +func (ah ArtifactHub) HandleDependents(comp model.ComponentDefinition, kc *kubernetes.Client, isDeploy, performUpgrade bool) (summary string, err error) { sourceURI, err := utils.Cast[string](comp.Metadata.AdditionalProperties["source_uri"]) // should be part of registrant data(?) if err != nil { return summary, err @@ -107,7 +103,7 @@ func (ah ArtifactHub) String() string { type Kubernetes struct{} -func (k Kubernetes) HandleDependents(_ v1beta1.ComponentDefinition, _ *kubernetes.Client, _, _ bool) (summary string, err error) { +func (k Kubernetes) HandleDependents(_ model.ComponentDefinition, _ *kubernetes.Client, _, _ bool) (summary string, err error) { return summary, err } diff --git a/models/meshmodel/core/v1beta1/policy.go b/models/meshmodel/core/v1beta1/policy.go index 03c4e417..1636e439 100644 --- a/models/meshmodel/core/v1beta1/policy.go +++ b/models/meshmodel/core/v1beta1/policy.go @@ -8,7 +8,7 @@ import ( "github.com/gofrs/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/utils" - "github.com/meshery/schemas/models/v1beta1" + v1beta1 "github.com/meshery/schemas/models/v1beta1/model" "gorm.io/gorm/clause" "github.com/layer5io/meshkit/models/meshmodel/entity" diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index c7f13311..a95192fb 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -8,9 +8,11 @@ import ( "github.com/gofrs/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" - model "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + models "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/meshery/schemas/models/v1beta1" + category "github.com/meshery/schemas/models/v1beta1/category" + connection "github.com/meshery/schemas/models/v1beta1/connection" + model "github.com/meshery/schemas/models/v1beta1/model" "golang.org/x/text/cases" "golang.org/x/text/language" "gorm.io/gorm/clause" @@ -24,9 +26,9 @@ import ( // 2. Entity type // 3. Entity type MeshModelRegistrantData struct { - Connection v1beta1.Connection `json:"connection"` - EntityType entity.EntityType `json:"entityType"` - Entity []byte `json:"entity"` //This will be type converted to appropriate entity on server based on passed entity type + Connection connection.Connection `json:"connection"` + EntityType entity.EntityType `json:"entityType"` + Entity []byte `json:"entity"` //This will be type converted to appropriate entity on server based on passed entity type } type Registry struct { ID uuid.UUID @@ -53,12 +55,12 @@ func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { } err := rm.db.AutoMigrate( &Registry{}, - &v1beta1.Connection{}, - &v1beta1.ComponentDefinition{}, + &connection.Connection{}, + &model.ComponentDefinition{}, &v1alpha2.RelationshipDefinition{}, - // &v1beta1.PolicyDefinition{}, - &v1beta1.ModelDefinition{}, - &v1beta1.CategoryDefinition{}, + // &models.PolicyDefinition{}, + &model.ModelDefinition{}, + &category.CategoryDefinition{}, ) if err != nil { return nil, err @@ -68,21 +70,23 @@ func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { func (rm *RegistryManager) Cleanup() { _ = rm.db.Migrator().DropTable( &Registry{}, - &v1beta1.Connection{}, - &v1beta1.ComponentDefinition{}, - &v1beta1.ModelDefinition{}, - &v1beta1.CategoryDefinition{}, + &connection.Connection{}, + &model.ComponentDefinition{}, + &model.ModelDefinition{}, + &category.CategoryDefinition{}, &v1alpha2.RelationshipDefinition{}, ) } -func (rm *RegistryManager) RegisterEntity(h v1beta1.Connection, en entity.Entity) (bool, bool, error) { +func (rm *RegistryManager) RegisterEntity(h connection.Connection, en entity.Entity) (bool, bool, error) { registrantID, err := h.Create(rm.db) if err != nil { + fmt.Println("REGISTRANT ERROR ") return true, false, err } entityID, err := en.Create(rm.db, registrantID) if err != nil { + fmt.Println("COMP ERROR ") return false, true, err } id, _ := uuid.NewV4() @@ -111,7 +115,7 @@ func (rm *RegistryManager) UpdateEntityStatus(ID string, status string, entityTy } switch entityType { case "models": - model := v1beta1.ModelDefinition{Id: entityID} + model := model.ModelDefinition{Id: entityID} err := model.UpdateStatus(rm.db, entity.EntityStatus(status)) if err != nil { return err @@ -122,33 +126,33 @@ func (rm *RegistryManager) UpdateEntityStatus(ID string, status string, entityTy } } -func (rm *RegistryManager) GetRegistrant(e entity.Entity) v1beta1.Connection { +func (rm *RegistryManager) GetRegistrant(e entity.Entity) connection.Connection { eID := e.GetID() var reg Registry _ = rm.db.Where("entity = ?", eID).Find(®).Error - var h v1beta1.Connection + var h connection.Connection _ = rm.db.Where("id = ?", reg.RegistrantID).Find(&h).Error return h } // to be removed -func (rm *RegistryManager) GetRegistrants(f *model.HostFilter) ([]model.MeshModelHostsWithEntitySummary, int64, error) { - var result []model.MesheryHostSummaryDB +func (rm *RegistryManager) GetRegistrants(f *models.HostFilter) ([]models.MeshModelHostsWithEntitySummary, int64, error) { + var result []models.MesheryHostSummaryDB var totalcount int64 db := rm.db - query := db.Table("hosts h"). + query := db.Table("connections c"). Count(&totalcount). - Select("h.id AS host_id, h.hostname, h.port, " + + Select("c.*, " + "COUNT(CASE WHEN r.type = 'component' THEN 1 END) AS components, " + "COUNT(CASE WHEN r.type = 'model' THEN 1 END) AS models," + "COUNT(CASE WHEN r.type = 'relationship' THEN 1 END) AS relationships, " + "COUNT(CASE WHEN r.type = 'policy' THEN 1 END) AS policies"). - Joins("LEFT JOIN registries r ON h.id = r.registrant_id"). - Group("h.id, h.hostname, h.port") + Joins("LEFT JOIN registries r ON c.id = r.registrant_id") + // Group("c.id, c.hostname, h.port") // required if f.DisplayName != "" { - query = query.Where("hostname LIKE ?", "%"+f.DisplayName+"%") + query = query.Where("kind LIKE ?", "%"+f.DisplayName+"%") } if f.OrderOn != "" { @@ -158,7 +162,7 @@ func (rm *RegistryManager) GetRegistrants(f *model.HostFilter) ([]model.MeshMode query = query.Order(f.OrderOn) } } else { - query = query.Order("hostname") + query = query.Order("kind") } query = query.Offset(f.Offset) @@ -172,14 +176,12 @@ func (rm *RegistryManager) GetRegistrants(f *model.HostFilter) ([]model.MeshMode return nil, 0, err } - var response []model.MeshModelHostsWithEntitySummary + var response []models.MeshModelHostsWithEntitySummary for _, r := range result { - res := model.MeshModelHostsWithEntitySummary{ - ID: r.HostID, - Hostname: HostnameToPascalCase(r.Hostname), - Port: r.Port, - Summary: model.EntitySummary{ + res := models.MeshModelHostsWithEntitySummary{ + Connection: r.Connection, + Summary: models.EntitySummary{ Models: r.Models, Components: r.Components, Relationships: r.Relationships, diff --git a/models/meshmodel/registry/v1beta1/category_filter.go b/models/meshmodel/registry/v1beta1/category_filter.go index d5258d6b..a2e17efa 100644 --- a/models/meshmodel/registry/v1beta1/category_filter.go +++ b/models/meshmodel/registry/v1beta1/category_filter.go @@ -3,7 +3,7 @@ package v1beta1 import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/meshery/schemas/models/v1beta1" + "github.com/meshery/schemas/models/v1beta1/category" "gorm.io/gorm/clause" ) @@ -35,7 +35,7 @@ func (cf *CategoryFilter) GetById(db *database.Handler) (entity.Entity, error) { } func (cf *CategoryFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { - var catdb []v1beta1.CategoryDefinition + var catdb []category.CategoryDefinition var cat []entity.Entity finder := db.Model(&catdb).Debug() diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index 1d8c3a2a..23c5fa57 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -1,9 +1,15 @@ package v1beta1 import ( + "fmt" + + "github.com/MUzairS15/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/meshery/schemas/models/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/registry" + "github.com/meshery/schemas/models/v1beta1/category" + "github.com/meshery/schemas/models/v1beta1/connection" + "github.com/meshery/schemas/models/v1beta1/model" "gorm.io/gorm/clause" ) @@ -26,10 +32,10 @@ type ComponentFilter struct { } type componentDefinitionWithModel struct { - ComponentDefinitionDB v1beta1.ComponentDefinition `gorm:"embedded"` - ModelDB v1beta1.ModelDefinition `gorm:"embedded"` - CategoryDB v1beta1.CategoryDefinition `gorm:"embedded"` - ConnectionDB v1beta1.Connection `gorm:"embedded"` + ComponentDefinitionDB model.ComponentDefinition `gorm:"embedded"` + ModelDB model.ModelDefinition `gorm:"embedded"` + CategoryDB category.CategoryDefinition `gorm:"embedded"` + ConnectionDB connection.Connection `gorm:"embedded"` } func (cf *ComponentFilter) GetById(db *database.Handler) (entity.Entity, error) { @@ -62,22 +68,20 @@ func countUniqueComponents(components []componentDefinitionWithModel) int { func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { var componentDefinitionsWithModel []componentDefinitionWithModel - finder := db.Model(&v1beta1.ComponentDefinition{}). - Select("component_definition_dbs.*, model_dbs.*,category_dbs.*, hosts.*"). + finder := db.Model(&model.ComponentDefinition{}). + Select("component_definition_dbs.*, model_dbs.*,category_dbs.*, connections.*"). Joins("JOIN model_dbs ON component_definition_dbs.model_id = model_dbs.id"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id"). - Joins("JOIN hosts ON hosts.id = model_dbs.host_id") - // + Joins("JOIN connections ON connections.id = model_dbs.host_id") // TODO(@MUzairS15): Refactor this once Status is made a first class field in ComponentFilter status := "enabled" - - if componentFilter.Status != "" { + + if componentFilter.Status != "" { status = componentFilter.Status } finder = finder.Where("model_dbs.status = ?", status) - if componentFilter.Greedy { if componentFilter.Name != "" && componentFilter.DisplayName != "" { finder = finder.Where("component_definition_dbs.component->>'kind' LIKE ? OR display_name LIKE ?", "%"+componentFilter.Name+"%", componentFilter.DisplayName+"%") @@ -124,8 +128,11 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti } else { finder = finder.Order("display_name") } + var count int64 finder.Count(&count) + fmt.Println("line 114 COUNT : =+++++++++++++++", count) + finder = finder.Offset(componentFilter.Offset) if componentFilter.Limit != 0 { finder = finder.Limit(componentFilter.Limit) @@ -133,6 +140,7 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti err := finder. Scan(&componentDefinitionsWithModel).Error if err != nil { + fmt.Println("line 123 : =+++++++++++++++", err) return nil, 0, 0, err } diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index 87c780de..4716ea04 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -1,11 +1,12 @@ package v1beta1 import ( + "fmt" + "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/models/meshmodel/registry" - "github.com/meshery/schemas/models/v1beta1" + v1beta1 "github.com/meshery/schemas/models/v1beta1/model" "gorm.io/gorm/clause" ) @@ -91,7 +92,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e finder := db.Model(&v1beta1.ModelDefinition{}).Preload("Category").Preload("Registrant"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id"). Joins("JOIN registries ON registries.entity = model_dbs.id"). - Joins("JOIN hosts ON hosts.id = registries.registrant_id") + Joins("JOIN connections ON connections.id = registries.registrant_id") // total count before pagination var count int64 @@ -99,6 +100,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e // include components and relationships in response body var includeComponents, includeRelationships bool + fmt.Println("line 65 : -------") if mf.Greedy { if mf.Name != "" && mf.DisplayName != "" { finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%") @@ -127,7 +129,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e finder = finder.Where("category_dbs.name = ?", mf.Category) } if mf.Registrant != "" { - finder = finder.Where("hosts.hostname = ?", mf.Registrant) + finder = finder.Where("connections.kind = ?", mf.Registrant) } if mf.Annotations == "true" { finder = finder.Where("model_dbs.metadata->>'isAnnotation' = true") @@ -167,6 +169,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e err := finder. Find(&modelWithCategories).Error if err != nil { + fmt.Println("line 131 : -------", err) return nil, 0, 0, err } diff --git a/utils/component/generator.go b/utils/component/generator.go index 535583f2..a4a9675a 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -9,6 +9,7 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1" + "github.com/meshery/schemas/models/v1beta1/model" ) const ComponentMetaNameKey = "name" @@ -46,11 +47,11 @@ var DefaultPathConfig2 = CuePathConfig{ var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2} -func Generate(crd string) (v1beta1.ComponentDefinition, error) { - component := v1beta1.ComponentDefinition{} +func Generate(crd string) (model.ComponentDefinition, error) { + component := model.ComponentDefinition{} component.SchemaVersion = v1beta1.ComponentSchemaVersion - component.Metadata = v1beta1.ComponentDefinition_Metadata{} + component.Metadata = model.ComponentDefinition_Metadata{} crdCue, err := utils.YamlToCue(crd) if err != nil { return component, err @@ -92,7 +93,7 @@ func Generate(crd string) (v1beta1.ComponentDefinition, error) { component.Component.Version = version } - component.Format = v1beta1.JSON + component.Format = model.JSON component.DisplayName = manifests.FormatToReadableString(name) return component, nil } diff --git a/utils/component/generator_test.go b/utils/component/generator_test.go index b0759fc9..31272db6 100644 --- a/utils/component/generator_test.go +++ b/utils/component/generator_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/layer5io/meshkit/utils/manifests" - "github.com/meshery/schemas/models/v1beta1" + v1beta1 "github.com/meshery/schemas/models/v1beta1/model" ) var istioCrd = ` diff --git a/utils/kubernetes/apply-manifest.go b/utils/kubernetes/apply-manifest.go index f1d1beac..91a4e8f9 100644 --- a/utils/kubernetes/apply-manifest.go +++ b/utils/kubernetes/apply-manifest.go @@ -2,6 +2,7 @@ package kubernetes import ( "context" + "fmt" "strings" v1 "k8s.io/api/core/v1" @@ -94,6 +95,7 @@ func (client *Client) ApplyManifest(contents []byte, recvOptions ApplyOptions) e return err } } else { + fmt.Println("Creating object", object) _, err = createObject(helper, options.Namespace, object, options.Update) if err != nil && !kubeerror.IsAlreadyExists(err) { if recvOptions.IgnoreErrors { diff --git a/validator/validate_test.go b/validator/validate_test.go index a0741736..76b99ed2 100644 --- a/validator/validate_test.go +++ b/validator/validate_test.go @@ -6,7 +6,9 @@ import ( "github.com/layer5io/meshkit/models/catalog/v1alpha1" "github.com/meshery/schemas/models/v1alpha2" - "github.com/meshery/schemas/models/v1beta1" + "github.com/meshery/schemas/models/v1beta1/model" + "github.com/meshery/schemas/models/v1beta1/category" + ) type ValidationCases struct { @@ -42,15 +44,15 @@ func TestValidator(t *testing.T) { }, { Path: "models", - Resource: v1beta1.ModelDefinition{ + Resource: model.ModelDefinition{ SchemaVersion: "v1beta1", Version: "1.0.0", - Category: v1beta1.CategoryDefinition{ + Category: category.CategoryDefinition{ Name: "test", }, - Model: v1beta1.Model{ + Model: model.Model{ Version: "1.0.0", }, Status: "", From bc016a0e49debce54e3cee19f7906d8b00ccc46b Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 6 Aug 2024 15:58:52 +0530 Subject: [PATCH 020/112] remove redundant package/functions Signed-off-by: MUzairS15 --- generators/artifacthub/package.go | 6 +-- generators/github/package.go | 6 +-- generators/models/interfaces.go | 4 +- go.mod | 2 +- go.sum | 4 +- .../core/v1beta1/application_component.go | 39 ------------------ models/meshmodel/core/v1beta1/host.go | 8 ++-- models/meshmodel/registry/registry.go | 15 ++++--- .../registry/v1beta1/component_filter.go | 13 +++--- .../registry/v1beta1/model_filter.go | 9 +++-- models/patterns/pattern.go | 18 ++------- utils/component/generator.go | 40 +++++++++---------- utils/component/generator_test.go | 8 ++-- utils/utils.go | 7 +--- validator/validate_test.go | 6 +-- 15 files changed, 68 insertions(+), 117 deletions(-) delete mode 100644 models/meshmodel/core/v1beta1/application_component.go diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index 02d5c68d..4e1ef920 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -10,7 +10,7 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" "github.com/layer5io/meshkit/utils/manifests" - "github.com/meshery/schemas/models/v1beta1/model" + _component "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/category" "gopkg.in/yaml.v2" ) @@ -37,8 +37,8 @@ func (pkg AhPackage) GetVersion() string { return pkg.Version } -func (pkg AhPackage) GenerateComponents() ([]model.ComponentDefinition, error) { - components := make([]model.ComponentDefinition, 0) +func (pkg AhPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { + components := make([]_component.ComponentDefinition, 0) // TODO: Move this to the configuration if pkg.ChartUrl == "" { diff --git a/generators/github/package.go b/generators/github/package.go index 42bc4e18..ef09aae8 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -8,7 +8,7 @@ import ( "github.com/layer5io/meshkit/utils/component" "github.com/layer5io/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1/category" - "github.com/meshery/schemas/models/v1beta1/model" + _component "github.com/meshery/schemas/models/v1beta1/component" ) type GitHubPackage struct { @@ -24,8 +24,8 @@ func (gp GitHubPackage) GetVersion() string { return gp.version } -func (gp GitHubPackage) GenerateComponents() ([]model.ComponentDefinition, error) { - components := make([]model.ComponentDefinition, 0) +func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { + components := make([]_component.ComponentDefinition, 0) data, err := os.ReadFile(gp.filePath) if err != nil { diff --git a/generators/models/interfaces.go b/generators/models/interfaces.go index 00987db5..6aa0c808 100644 --- a/generators/models/interfaces.go +++ b/generators/models/interfaces.go @@ -1,6 +1,6 @@ package models -import v1beta1 "github.com/meshery/schemas/models/v1beta1/model" +import "github.com/meshery/schemas/models/v1beta1/component" // anything that can be validated is a Validator type Validator interface { @@ -11,7 +11,7 @@ type Validator interface { // system's capabilities in Meshery // A Package should have all the information that we need to generate the components type Package interface { - GenerateComponents() ([]v1beta1.ComponentDefinition, error) + GenerateComponents() ([]component.ComponentDefinition, error) GetVersion() string } diff --git a/go.mod b/go.mod index 3b2f4238..7b0ba119 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/postgres v1.5.3 gorm.io/driver/sqlite v1.5.4 - gorm.io/gorm v1.25.5 + gorm.io/gorm v1.25.11 helm.sh/helm/v3 v3.13.2 k8s.io/api v0.28.4 k8s.io/apimachinery v0.28.4 diff --git a/go.sum b/go.sum index ef6fa25a..356c54af 100644 --- a/go.sum +++ b/go.sum @@ -1198,8 +1198,8 @@ gorm.io/driver/postgres v1.5.3 h1:qKGY5CPHOuj47K/VxbCXJfFvIUeqMSXXadqdCY+MbBU= gorm.io/driver/postgres v1.5.3/go.mod h1:F+LtvlFhZT7UBiA81mC9W6Su3D4WUhSboc/36QZU0gk= gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0= gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= -gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= -gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= +gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= diff --git a/models/meshmodel/core/v1beta1/application_component.go b/models/meshmodel/core/v1beta1/application_component.go deleted file mode 100644 index 1e96a0f7..00000000 --- a/models/meshmodel/core/v1beta1/application_component.go +++ /dev/null @@ -1,39 +0,0 @@ -package v1beta1 - -// // ComponentParameter is the structure for the core Application Component -// // Paramater -// type ComponentParameter struct { -// Name string `json:"name"` -// FieldPaths []string `json:"fieldPaths"` -// Required *bool `json:"required,omitempty"` -// Description *string `json:"description,omitempty"` -// } - -// func GetAPIVersionFromComponent(comp model.ComponentDefinition) string { -// return comp.Annotations[MesheryAnnotationPrefix+".k8s.APIVersion"] -// } - -// func GetKindFromComponent(comp model.ComponentDefinition) string { -// kind := strings.TrimPrefix(comp.Annotations[MesheryAnnotationPrefix+".k8s.Kind"], "/") -// return kind -// } - -// func GetAnnotationsForWorkload(w model.ComponentDefinition) map[string]string { -// res := map[string]string{} - -// for key, val := range w.Metadata { -// if v, ok := val.(string); ok { -// res[strings.ReplaceAll(fmt.Sprintf("%s.%s", MesheryAnnotationPrefix, key), " ", "")] = v -// } -// } -// sourceURI, ok := w.Model.Metadata["source_uri"].(string) -// if ok { -// res[fmt.Sprintf("%s.model.source_uri", MesheryAnnotationPrefix)] = sourceURI -// } -// res[fmt.Sprintf("%s.model.name", MesheryAnnotationPrefix)] = w.Model.Name -// res[fmt.Sprintf("%s.k8s.APIVersion", MesheryAnnotationPrefix)] = w.Component.Version -// res[fmt.Sprintf("%s.k8s.Kind", MesheryAnnotationPrefix)] = w.Component.Kind -// res[fmt.Sprintf("%s.model.version", MesheryAnnotationPrefix)] = w.Model.Version -// res[fmt.Sprintf("%s.model.category", MesheryAnnotationPrefix)] = w.Model.Category.Name -// return res -// } diff --git a/models/meshmodel/core/v1beta1/host.go b/models/meshmodel/core/v1beta1/host.go index 52acf1bb..fa5da2a5 100644 --- a/models/meshmodel/core/v1beta1/host.go +++ b/models/meshmodel/core/v1beta1/host.go @@ -6,7 +6,7 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/kubernetes" "github.com/meshery/schemas/models/v1beta1/connection" - "github.com/meshery/schemas/models/v1beta1/model" + "github.com/meshery/schemas/models/v1beta1/component" ) type MeshModelHostsWithEntitySummary struct { @@ -40,7 +40,7 @@ type HostFilter struct { } type DependencyHandler interface { - HandleDependents(comp model.ComponentDefinition, kc *kubernetes.Client, isDeploy, performUpgrade bool) (string, error) + HandleDependents(comp component.ComponentDefinition, kc *kubernetes.Client, isDeploy, performUpgrade bool) (string, error) String() string } @@ -61,7 +61,7 @@ type ArtifactHub struct{} const MesheryAnnotationPrefix = "design.meshmodel.io" -func (ah ArtifactHub) HandleDependents(comp model.ComponentDefinition, kc *kubernetes.Client, isDeploy, performUpgrade bool) (summary string, err error) { +func (ah ArtifactHub) HandleDependents(comp component.ComponentDefinition, kc *kubernetes.Client, isDeploy, performUpgrade bool) (summary string, err error) { sourceURI, err := utils.Cast[string](comp.Metadata.AdditionalProperties["source_uri"]) // should be part of registrant data(?) if err != nil { return summary, err @@ -103,7 +103,7 @@ func (ah ArtifactHub) String() string { type Kubernetes struct{} -func (k Kubernetes) HandleDependents(_ model.ComponentDefinition, _ *kubernetes.Client, _, _ bool) (summary string, err error) { +func (k Kubernetes) HandleDependents(_ component.ComponentDefinition, _ *kubernetes.Client, _, _ bool) (summary string, err error) { return summary, err } diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index a95192fb..852dea86 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -10,9 +10,11 @@ import ( "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" models "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" - category "github.com/meshery/schemas/models/v1beta1/category" - connection "github.com/meshery/schemas/models/v1beta1/connection" - model "github.com/meshery/schemas/models/v1beta1/model" + "github.com/meshery/schemas/models/v1beta1/category" + "github.com/meshery/schemas/models/v1beta1/component" + "github.com/meshery/schemas/models/v1beta1/connection" + "github.com/meshery/schemas/models/v1beta1/model" + "github.com/meshery/schemas/models/v1alpha3/relationship" "golang.org/x/text/cases" "golang.org/x/text/language" "gorm.io/gorm/clause" @@ -56,7 +58,7 @@ func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { err := rm.db.AutoMigrate( &Registry{}, &connection.Connection{}, - &model.ComponentDefinition{}, + &component.ComponentDefinition{}, &v1alpha2.RelationshipDefinition{}, // &models.PolicyDefinition{}, &model.ModelDefinition{}, @@ -71,10 +73,10 @@ func (rm *RegistryManager) Cleanup() { _ = rm.db.Migrator().DropTable( &Registry{}, &connection.Connection{}, - &model.ComponentDefinition{}, + &component.ComponentDefinition{}, &model.ModelDefinition{}, &category.CategoryDefinition{}, - &v1alpha2.RelationshipDefinition{}, + &relationship.RelationshipDefinition{}, ) } func (rm *RegistryManager) RegisterEntity(h connection.Connection, en entity.Entity) (bool, bool, error) { @@ -178,6 +180,7 @@ func (rm *RegistryManager) GetRegistrants(f *models.HostFilter) ([]models.MeshMo var response []models.MeshModelHostsWithEntitySummary + fmt.Println("result--------------------------", result) for _, r := range result { res := models.MeshModelHostsWithEntitySummary{ Connection: r.Connection, diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index 23c5fa57..aff5b9cf 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -8,6 +8,7 @@ import ( "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/meshery/schemas/models/v1beta1/category" + "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/connection" "github.com/meshery/schemas/models/v1beta1/model" "gorm.io/gorm/clause" @@ -32,10 +33,10 @@ type ComponentFilter struct { } type componentDefinitionWithModel struct { - ComponentDefinitionDB model.ComponentDefinition `gorm:"embedded"` - ModelDB model.ModelDefinition `gorm:"embedded"` - CategoryDB category.CategoryDefinition `gorm:"embedded"` - ConnectionDB connection.Connection `gorm:"embedded"` + ComponentDefinitionDB component.ComponentDefinition `gorm:"embedded"` + ModelDB model.ModelDefinition `gorm:"embedded"` + CategoryDB category.CategoryDefinition `gorm:"embedded"` + ConnectionDB connection.Connection `gorm:"embedded"` } func (cf *ComponentFilter) GetById(db *database.Handler) (entity.Entity, error) { @@ -68,11 +69,11 @@ func countUniqueComponents(components []componentDefinitionWithModel) int { func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { var componentDefinitionsWithModel []componentDefinitionWithModel - finder := db.Model(&model.ComponentDefinition{}). + finder := db.Model(&component.ComponentDefinition{}). Select("component_definition_dbs.*, model_dbs.*,category_dbs.*, connections.*"). Joins("JOIN model_dbs ON component_definition_dbs.model_id = model_dbs.id"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id"). - Joins("JOIN connections ON connections.id = model_dbs.host_id") + Joins("JOIN connections ON connections.id = model_dbs.connection_id") // TODO(@MUzairS15): Refactor this once Status is made a first class field in ComponentFilter status := "enabled" diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index 4716ea04..2dd3999c 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -6,6 +6,8 @@ import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/meshmodel/registry" + "github.com/meshery/schemas/models/v1beta1/model" v1beta1 "github.com/meshery/schemas/models/v1beta1/model" "gorm.io/gorm/clause" @@ -87,9 +89,10 @@ func (mf *ModelFilter) GetById(db *database.Handler) (entity.Entity, error) { } func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { - var modelWithCategories []v1beta1.ModelDefinition - finder := db.Model(&v1beta1.ModelDefinition{}).Preload("Category").Preload("Registrant"). + var modelWithCategories []model.ModelDefinition + + finder := db.Model(&model.ModelDefinition{}).Preload("Category").Preload("Registrant"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id"). Joins("JOIN registries ON registries.entity = model_dbs.id"). Joins("JOIN connections ON connections.id = registries.registrant_id") @@ -196,7 +199,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e if err := finder.Scan(&relationships).Error; err != nil { return nil, 0, 0, err } - // _modelDB.Relationships = relationships FIX THIS + _modelDB.Relationships = relationships } defs = append(defs, &_modelDB) } diff --git a/models/patterns/pattern.go b/models/patterns/pattern.go index 4b76a22b..04fe2c85 100644 --- a/models/patterns/pattern.go +++ b/models/patterns/pattern.go @@ -2,12 +2,10 @@ package patterns import ( "github.com/Masterminds/semver/v3" - "github.com/layer5io/meshkit/utils" - "github.com/meshery/schemas/models/v1alpha2" - "gopkg.in/yaml.v2" + "github.com/meshery/schemas/models/v1beta1/pattern" ) -func GetNextVersion(p *v1alpha2.PatternFile) (string, error) { +func GetNextVersion(p *pattern.PatternFile) (string, error) { // Existing patterns do not have version hence when trying to assign next version for such patterns, it will fail the validation. // Hence, if version is not present, start versioning for those afresh. if p.Version == "" { @@ -24,16 +22,6 @@ func GetNextVersion(p *v1alpha2.PatternFile) (string, error) { return nextVersion, nil } -func AssignVersion(p *v1alpha2.PatternFile) { +func AssignVersion(p *pattern.PatternFile) { p.Version = semver.New(0, 0, 1, "", "").String() } - -func GetPatternFormat(patternFile string) (*v1alpha2.PatternFile, error) { - pattern := v1alpha2.PatternFile{} - err := yaml.Unmarshal([]byte(patternFile), &pattern) - if err != nil { - err = utils.ErrDecodeYaml(err) - return nil, err - } - return &pattern, nil -} diff --git a/utils/component/generator.go b/utils/component/generator.go index a4a9675a..417d59cd 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -9,7 +9,7 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1" - "github.com/meshery/schemas/models/v1beta1/model" + "github.com/meshery/schemas/models/v1beta1/component" ) const ComponentMetaNameKey = "name" @@ -47,14 +47,14 @@ var DefaultPathConfig2 = CuePathConfig{ var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2} -func Generate(crd string) (model.ComponentDefinition, error) { - component := model.ComponentDefinition{} - component.SchemaVersion = v1beta1.ComponentSchemaVersion +func Generate(crd string) (component.ComponentDefinition, error) { + cmp := component.ComponentDefinition{} + cmp.SchemaVersion = v1beta1.ComponentSchemaVersion - component.Metadata = model.ComponentDefinition_Metadata{} + cmp.Metadata = component.ComponentDefinition_Metadata{} crdCue, err := utils.YamlToCue(crd) if err != nil { - return component, err + return cmp, err } var schema string for _, cfg := range Configs { @@ -63,39 +63,39 @@ func Generate(crd string) (model.ComponentDefinition, error) { break } } - component.Component.Schema = schema + cmp.Component.Schema = schema name, err := extractCueValueFromPath(crdCue, DefaultPathConfig.NamePath) if err != nil { - return component, err + return cmp, err } version, err := extractCueValueFromPath(crdCue, DefaultPathConfig.VersionPath) if err != nil { - return component, err + return cmp, err } group, err := extractCueValueFromPath(crdCue, DefaultPathConfig.GroupPath) if err != nil { - return component, err + return cmp, err } // return component, err Ignore error if scope isn't found - if component.Metadata.AdditionalProperties == nil { - component.Metadata.AdditionalProperties = make(map[string]interface{}) + if cmp.Metadata.AdditionalProperties == nil { + cmp.Metadata.AdditionalProperties = make(map[string]interface{}) } scope, _ := extractCueValueFromPath(crdCue, DefaultPathConfig.ScopePath) if scope == "Cluster" { - component.Metadata.AdditionalProperties["isNamespaced"] = false + cmp.Metadata.AdditionalProperties["isNamespaced"] = false } else if scope == "Namespaced" { - component.Metadata.AdditionalProperties["isNamespaced"] = true + cmp.Metadata.AdditionalProperties["isNamespaced"] = true } - component.Component.Kind = name + cmp.Component.Kind = name if group != "" { - component.Component.Version = fmt.Sprintf("%s/%s", group, version) + cmp.Component.Version = fmt.Sprintf("%s/%s", group, version) } else { - component.Component.Version = version + cmp.Component.Version = version } - component.Format = model.JSON - component.DisplayName = manifests.FormatToReadableString(name) - return component, nil + cmp.Format = component.JSON + cmp.DisplayName = manifests.FormatToReadableString(name) + return cmp, nil } /* diff --git a/utils/component/generator_test.go b/utils/component/generator_test.go index 31272db6..a276e0a4 100644 --- a/utils/component/generator_test.go +++ b/utils/component/generator_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/layer5io/meshkit/utils/manifests" - v1beta1 "github.com/meshery/schemas/models/v1beta1/model" + "github.com/meshery/schemas/models/v1beta1/component" ) var istioCrd = ` @@ -124,8 +124,8 @@ spec: status: {} ` -func getNewComponent(spec string, name string, version string) v1beta1.ComponentDefinition { - comp := v1beta1.ComponentDefinition{} +func getNewComponent(spec string, name string, version string) component.ComponentDefinition { + comp := component.ComponentDefinition{} comp.Component.Schema = spec comp.DisplayName = manifests.FormatToReadableString(name) comp.Component.Version = version @@ -136,7 +136,7 @@ func getNewComponent(spec string, name string, version string) v1beta1.Component func TestGenerate(t *testing.T) { var tests = []struct { crd string - want v1beta1.ComponentDefinition + want component.ComponentDefinition }{ {istioCrd, getNewComponent("", "WasmPlugin", "extensions.istio.io/v1beta1")}, } diff --git a/utils/utils.go b/utils/utils.go index c1f41a5d..02707a3f 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -411,12 +411,7 @@ func IsInterfaceNil(val interface{}) bool { if val == nil { return true } - switch reflect.TypeOf(val).Kind() { - case reflect.Ptr, reflect.Map, reflect.Array, reflect.Chan, reflect.Slice: - return reflect.ValueOf(val).IsNil() - } - return false - + return reflect.ValueOf(val).IsZero() } func IsSchemaEmpty(schema string) (valid bool) { diff --git a/validator/validate_test.go b/validator/validate_test.go index 76b99ed2..48303d22 100644 --- a/validator/validate_test.go +++ b/validator/validate_test.go @@ -6,9 +6,9 @@ import ( "github.com/layer5io/meshkit/models/catalog/v1alpha1" "github.com/meshery/schemas/models/v1alpha2" - "github.com/meshery/schemas/models/v1beta1/model" + "github.com/meshery/schemas/models/v1beta1" "github.com/meshery/schemas/models/v1beta1/category" - + "github.com/meshery/schemas/models/v1beta1/model" ) type ValidationCases struct { @@ -46,7 +46,7 @@ func TestValidator(t *testing.T) { Path: "models", Resource: model.ModelDefinition{ - SchemaVersion: "v1beta1", + SchemaVersion: v1beta1.ModelSchemaVersion, Version: "1.0.0", Category: category.CategoryDefinition{ From 40f9856c0c6e9898533b4d253848b54c1c7f264b Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 7 Aug 2024 18:16:03 +0530 Subject: [PATCH 021/112] add debug statements Signed-off-by: MUzairS15 --- models/patterns/pattern.go | 11 +++++++++++ utils/cue.go | 5 +++++ utils/utils.go | 2 ++ validator/validate.go | 19 +++++++++++++++---- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/models/patterns/pattern.go b/models/patterns/pattern.go index 04fe2c85..5d76bb8b 100644 --- a/models/patterns/pattern.go +++ b/models/patterns/pattern.go @@ -2,7 +2,9 @@ package patterns import ( "github.com/Masterminds/semver/v3" + "github.com/layer5io/meshkit/utils" "github.com/meshery/schemas/models/v1beta1/pattern" + "gopkg.in/yaml.v2" ) func GetNextVersion(p *pattern.PatternFile) (string, error) { @@ -25,3 +27,12 @@ func GetNextVersion(p *pattern.PatternFile) (string, error) { func AssignVersion(p *pattern.PatternFile) { p.Version = semver.New(0, 0, 1, "", "").String() } +func GetPatternFormat(patternFile string) (*pattern.PatternFile, error) { + pattern := pattern.PatternFile{} + err := yaml.Unmarshal([]byte(patternFile), &pattern) + if err != nil { + err = utils.ErrDecodeYaml(err) + return nil, err + } + return &pattern, nil +} diff --git a/utils/cue.go b/utils/cue.go index c7854676..5c34bde6 100644 --- a/utils/cue.go +++ b/utils/cue.go @@ -90,14 +90,19 @@ func JsonSchemaToCue(value string) (cue.Value, error) { extractedSchema, err := jsonschema.Extract(cueJsonSchemaExpr, &jsonschema.Config{ PkgName: "jsonschemeconv", }) + + s := errors.Details(err, nil) + fmt.Println("line 95 : ==========", s, err) if err != nil { return out, ErrJsonSchemaToCue(err) } src, err := format.Node(extractedSchema) + fmt.Println("line 100 : ==========", err) if err != nil { return out, ErrJsonSchemaToCue(err) } out = cueCtx.CompileString(string(src)) + fmt.Println("line 105 : ==========", err) if out.Err() != nil { return out, ErrJsonSchemaToCue(out.Err()) } diff --git a/utils/utils.go b/utils/utils.go index 02707a3f..0fc2e86c 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -408,9 +408,11 @@ func ExtractDomainFromURL(location string) string { } func IsInterfaceNil(val interface{}) bool { + fmt.Println("TEST 441 : ", val == nil) if val == nil { return true } + fmt.Println("TEST 415 : ", reflect.ValueOf(val).IsZero()) return reflect.ValueOf(val).IsZero() } diff --git a/validator/validate.go b/validator/validate.go index a748d4f8..06fa894e 100644 --- a/validator/validate.go +++ b/validator/validate.go @@ -8,6 +8,7 @@ import ( "cuelang.org/go/cue" cueerrors "cuelang.org/go/cue/errors" "github.com/layer5io/meshkit/errors" + "github.com/layer5io/meshkit/logger" "github.com/layer5io/meshkit/utils" "github.com/meshery/schemas" ) @@ -30,9 +31,9 @@ func loadSchema() error { return nil } - file, err := schemas.Schemas.Open("schemas/openapi.yml") + file, err := schemas.Schemas.Open("schemas/constructs/v1beta1/designs.json") if err != nil { - return utils.ErrReadFile(err, "schemas/openapi.yml") + return utils.ErrReadFile(err, "schemas/constructs/v1beta1/designs.json") } cueschema, err = utils.ConvertoCue(file) @@ -44,7 +45,7 @@ func loadSchema() error { func GetSchemaFor(resourceName string) (cue.Value, error) { var schema cue.Value - schemaPathForResource := fmt.Sprintf("%s.%s", schemaPath, resourceName) + schemaPathForResource := "" err := loadSchema() if err != nil { @@ -61,7 +62,12 @@ func GetSchemaFor(resourceName string) (cue.Value, error) { return schema, utils.ErrMarshal(err) } + schema, err = utils.JsonSchemaToCue(string(byt)) + + log, _ := logger.New("test-validate", logger.Options{}) + log.Error(err) + fmt.Println("line 64 ===========================", err) if err != nil { return schema, err } @@ -83,10 +89,12 @@ func Validate(schema cue.Value, resourceValue interface{}) error { valid, errs := utils.Validate(schema, cv) if !valid { + errs := convertCueErrorsToStrings(errs) + // fmt.Println("line 93 ===========================", errs) return errors.New(ErrValidateCode, errors.Alert, []string{"validation for the resource failed"}, - convertCueErrorsToStrings(errs), + errs, []string{}, []string{}, ) } @@ -96,9 +104,12 @@ func Validate(schema cue.Value, resourceValue interface{}) error { func convertCueErrorsToStrings(errs []cueerrors.Error) []string { var res []string for _, err := range errs { + _ = cueerrors.Sanitize(err) } for _, err2 := range errs { + fmt.Println("\n\n[line 1111 ===========================]: ", err2.Error()) + res = append(res, err2.Error()) } return res From 079f7b0654d02f51d443a8ce16e179d494861e9d Mon Sep 17 00:00:00 2001 From: Mohd Uzair Date: Mon, 12 Aug 2024 20:49:43 +0530 Subject: [PATCH 022/112] Apply suggestions from code review Signed-off-by: Mohd Uzair --- models/meshmodel/core/policies/rego_policy_relationship.go | 1 - models/meshmodel/registry/v1beta1/component_filter.go | 1 - models/meshmodel/registry/v1beta1/model_filter.go | 2 -- utils/cue.go | 3 --- utils/utils.go | 1 - validator/validate.go | 3 --- 6 files changed, 11 deletions(-) diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index 0e258e7f..423116d3 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -28,7 +28,6 @@ func NewRegoInstance(policyDir string, regManager *registry.RegistryManager) (*R ctx := context.Background() registeredRelationships, _, _, err := regManager.GetEntities(&v1alpha2.RelationshipFilter{}) if err != nil { - fmt.Println(err, ";;s;d;sd") return nil, err } diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index aff5b9cf..12eb35ce 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -132,7 +132,6 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti var count int64 finder.Count(&count) - fmt.Println("line 114 COUNT : =+++++++++++++++", count) finder = finder.Offset(componentFilter.Offset) if componentFilter.Limit != 0 { diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index 2dd3999c..4dd5019d 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -103,7 +103,6 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e // include components and relationships in response body var includeComponents, includeRelationships bool - fmt.Println("line 65 : -------") if mf.Greedy { if mf.Name != "" && mf.DisplayName != "" { finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%") @@ -172,7 +171,6 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e err := finder. Find(&modelWithCategories).Error if err != nil { - fmt.Println("line 131 : -------", err) return nil, 0, 0, err } diff --git a/utils/cue.go b/utils/cue.go index 5c34bde6..68a6b036 100644 --- a/utils/cue.go +++ b/utils/cue.go @@ -92,17 +92,14 @@ func JsonSchemaToCue(value string) (cue.Value, error) { }) s := errors.Details(err, nil) - fmt.Println("line 95 : ==========", s, err) if err != nil { return out, ErrJsonSchemaToCue(err) } src, err := format.Node(extractedSchema) - fmt.Println("line 100 : ==========", err) if err != nil { return out, ErrJsonSchemaToCue(err) } out = cueCtx.CompileString(string(src)) - fmt.Println("line 105 : ==========", err) if out.Err() != nil { return out, ErrJsonSchemaToCue(out.Err()) } diff --git a/utils/utils.go b/utils/utils.go index 0fc2e86c..64d17c4e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -412,7 +412,6 @@ func IsInterfaceNil(val interface{}) bool { if val == nil { return true } - fmt.Println("TEST 415 : ", reflect.ValueOf(val).IsZero()) return reflect.ValueOf(val).IsZero() } diff --git a/validator/validate.go b/validator/validate.go index 06fa894e..024a2ef1 100644 --- a/validator/validate.go +++ b/validator/validate.go @@ -67,7 +67,6 @@ func GetSchemaFor(resourceName string) (cue.Value, error) { log, _ := logger.New("test-validate", logger.Options{}) log.Error(err) - fmt.Println("line 64 ===========================", err) if err != nil { return schema, err } @@ -90,7 +89,6 @@ func Validate(schema cue.Value, resourceValue interface{}) error { valid, errs := utils.Validate(schema, cv) if !valid { errs := convertCueErrorsToStrings(errs) - // fmt.Println("line 93 ===========================", errs) return errors.New(ErrValidateCode, errors.Alert, []string{"validation for the resource failed"}, @@ -108,7 +106,6 @@ func convertCueErrorsToStrings(errs []cueerrors.Error) []string { _ = cueerrors.Sanitize(err) } for _, err2 := range errs { - fmt.Println("\n\n[line 1111 ===========================]: ", err2.Error()) res = append(res, err2.Error()) } From 3a2eeb3a6172c9b58642f1eee83322cb8c4b6c8c Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 12 Aug 2024 20:50:57 +0530 Subject: [PATCH 023/112] remove old constructs Signed-off-by: MUzairS15 --- .../meshmodel/core/v1alpha2/relationship.go | 81 ------------------- models/meshmodel/core/v1beta1/host.go | 2 +- models/meshmodel/registry/registry.go | 7 +- .../registry/v1alpha2/relationship_filter.go | 8 +- .../registry/v1beta1/model_filter.go | 8 +- 5 files changed, 12 insertions(+), 94 deletions(-) delete mode 100644 models/meshmodel/core/v1alpha2/relationship.go diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go deleted file mode 100644 index fc823bee..00000000 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ /dev/null @@ -1,81 +0,0 @@ -package v1alpha2 - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/gofrs/uuid" - "github.com/layer5io/meshkit/database" - v1beta1 "github.com/meshery/schemas/models/v1beta1/model" - - "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/utils" - "gorm.io/gorm/clause" -) - -const RelationshipSchemaVersion = "relationships.meshery.io/v1alpha2" - -type RelationshipDefinition struct { - ID uuid.UUID `json:"id"` - SchemaVersion string `json:"schemaVersion,omitempty" yaml:"schemaVersion"` - Version string `json:"version,omitempty" yaml:"version"` - Kind string `json:"kind,omitempty" yaml:"kind"` - // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. - // Though, the column name and the json representation is "type". - RelationshipType string `json:"type" yaml:"type" gorm:"type"` - SubType string `json:"subType" yaml:"subType"` - EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata" gorm:"type:bytes;serializer:json"` - ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:model_id"` - Model v1beta1.ModelDefinition `json:"model" gorm:"foreignKey:ModelID;references:Id"` - Selectors []map[string]interface{} `json:"selectors" yaml:"selectors" gorm:"type:bytes;serializer:json"` -} - -func (r RelationshipDefinition) TableName() string { - return "relationship_definition_dbs" -} - -func (r RelationshipDefinition) Type() entity.EntityType { - return entity.RelationshipDefinition -} - -func (r *RelationshipDefinition) GenerateID() (uuid.UUID, error) { - return uuid.NewV4() -} - -func (r RelationshipDefinition) GetID() uuid.UUID { - return r.ID -} - -func (r *RelationshipDefinition) GetEntityDetail() string { - return fmt.Sprintf("type: %s, definition version: %s, kind: %s, model: %s, version: %s", r.Type(), r.Version, r.Kind, r.Model.Name, r.Model.Version) -} - -func (r *RelationshipDefinition) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error) { - r.ID, _ = r.GenerateID() - mid, err := r.Model.Create(db, hostID) - if err != nil { - return uuid.UUID{}, err - } - r.ModelID = mid - err = db.Omit(clause.Associations).Create(&r).Error - if err != nil { - return uuid.UUID{}, err - } - return r.ID, err -} - -func (m *RelationshipDefinition) UpdateStatus(db *database.Handler, status entity.EntityStatus) error { - return nil -} - -func (c RelationshipDefinition) WriteComponentDefinition(relDirPath string) error { - relPath := filepath.Join(relDirPath, c.Kind, string(c.Type())+".json") - err := utils.WriteJSONToFile[RelationshipDefinition](relPath, c) - return err -} - -func (r *RelationshipDefinition) GetDefaultEvaluationQuery() string { - return fmt.Sprintf("%s_%s_relationship", strings.ToLower(r.Kind), strings.ToLower(r.SubType)) -} diff --git a/models/meshmodel/core/v1beta1/host.go b/models/meshmodel/core/v1beta1/host.go index fa5da2a5..0ac38d14 100644 --- a/models/meshmodel/core/v1beta1/host.go +++ b/models/meshmodel/core/v1beta1/host.go @@ -5,8 +5,8 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/kubernetes" - "github.com/meshery/schemas/models/v1beta1/connection" "github.com/meshery/schemas/models/v1beta1/component" + "github.com/meshery/schemas/models/v1beta1/connection" ) type MeshModelHostsWithEntitySummary struct { diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index 852dea86..476a9055 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -7,14 +7,13 @@ import ( "github.com/gofrs/uuid" "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" models "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/meshery/schemas/models/v1alpha3/relationship" "github.com/meshery/schemas/models/v1beta1/category" "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/connection" "github.com/meshery/schemas/models/v1beta1/model" - "github.com/meshery/schemas/models/v1alpha3/relationship" "golang.org/x/text/cases" "golang.org/x/text/language" "gorm.io/gorm/clause" @@ -59,8 +58,8 @@ func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { &Registry{}, &connection.Connection{}, &component.ComponentDefinition{}, - &v1alpha2.RelationshipDefinition{}, - // &models.PolicyDefinition{}, + &relationship.RelationshipDefinition{}, + &models.PolicyDefinition{}, &model.ModelDefinition{}, &category.CategoryDefinition{}, ) diff --git a/models/meshmodel/registry/v1alpha2/relationship_filter.go b/models/meshmodel/registry/v1alpha2/relationship_filter.go index b9ac33ab..b9b85bcb 100644 --- a/models/meshmodel/registry/v1alpha2/relationship_filter.go +++ b/models/meshmodel/registry/v1alpha2/relationship_filter.go @@ -2,9 +2,9 @@ package v1alpha2 import ( "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/models/meshmodel/registry" + "github.com/meshery/schemas/models/v1alpha3/relationship" + "gorm.io/gorm/clause" ) @@ -45,8 +45,8 @@ func (rf *RelationshipFilter) GetById(db *database.Handler) (entity.Entity, erro func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { - var relationshipDefinitionsWithModel []v1alpha2.RelationshipDefinition - finder := db.Model(&v1alpha2.RelationshipDefinition{}).Preload("Model").Preload("Model.Category"). + var relationshipDefinitionsWithModel []relationship.RelationshipDefinition + finder := db.Model(&relationship.RelationshipDefinition{}).Preload("Model").Preload("Model.Category"). Joins("JOIN model_dbs ON relationship_definition_dbs.model_id = model_dbs.id"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index 4dd5019d..c003c7ab 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -4,9 +4,9 @@ import ( "fmt" "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/models/meshmodel/registry" + "github.com/meshery/schemas/models/v1alpha3/relationship" + "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/model" v1beta1 "github.com/meshery/schemas/models/v1beta1/model" @@ -190,8 +190,8 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e _modelDB.Components = components } if includeRelationships { - var relationships []v1alpha2.RelationshipDefinition - finder := db.Model(&v1alpha2.RelationshipDefinition{}). + var relationships []relationship.RelationshipDefinition + finder := db.Model(&relationship.RelationshipDefinition{}). Select("relationship_definition_dbs.*"). Where("relationship_definition_dbs.model_id = ?", _modelDB.Id) if err := finder.Scan(&relationships).Error; err != nil { From 24209aea68475c0e5727c4e996e3dcc4533dc64a Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 12 Aug 2024 20:51:43 +0530 Subject: [PATCH 024/112] remove unused imports Signed-off-by: MUzairS15 --- models/meshmodel/registry/v1beta1/model_filter.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index c003c7ab..cc45197e 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -1,8 +1,6 @@ package v1beta1 import ( - "fmt" - "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/meshery/schemas/models/v1alpha3/relationship" From 2468ab15881ad000026293f01f32a44d4b1c4f32 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 12 Aug 2024 21:34:50 +0530 Subject: [PATCH 025/112] fix merge conflicts Signed-off-by: MUzairS15 --- .../core/policies/rego_policy_relationship.go | 4 +- .../relationship_filter.go | 5 +- .../registry/v1beta1/category_filter.go | 9 +- .../registry/v1beta1/component_filter.go | 3 +- .../registry/v1beta1/model_filter.go | 24 ++-- models/registration/dir.go | 78 +++++------ models/registration/register.go | 57 +++++--- models/registration/svg_helper.go | 131 +++++++++--------- models/registration/utils.go | 65 ++++----- utils/cue.go | 1 - validator/validate.go | 6 +- 11 files changed, 202 insertions(+), 181 deletions(-) rename models/meshmodel/registry/{v1alpha2 => v1alpha3}/relationship_filter.go (96%) diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index 423116d3..88fa8951 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/layer5io/meshkit/models/meshmodel/registry" - "github.com/layer5io/meshkit/models/meshmodel/registry/v1alpha2" + "github.com/layer5io/meshkit/models/meshmodel/registry/v1alpha3" "github.com/layer5io/meshkit/utils" "github.com/open-policy-agent/opa/rego" "github.com/open-policy-agent/opa/storage" @@ -26,7 +26,7 @@ func NewRegoInstance(policyDir string, regManager *registry.RegistryManager) (*R var store storage.Store ctx := context.Background() - registeredRelationships, _, _, err := regManager.GetEntities(&v1alpha2.RelationshipFilter{}) + registeredRelationships, _, _, err := regManager.GetEntities(&v1alpha3.RelationshipFilter{}) if err != nil { return nil, err } diff --git a/models/meshmodel/registry/v1alpha2/relationship_filter.go b/models/meshmodel/registry/v1alpha3/relationship_filter.go similarity index 96% rename from models/meshmodel/registry/v1alpha2/relationship_filter.go rename to models/meshmodel/registry/v1alpha3/relationship_filter.go index b9b85bcb..e3893ce9 100644 --- a/models/meshmodel/registry/v1alpha2/relationship_filter.go +++ b/models/meshmodel/registry/v1alpha3/relationship_filter.go @@ -1,8 +1,9 @@ -package v1alpha2 +package v1alpha3 import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/meshery/schemas/models/v1alpha3/relationship" "gorm.io/gorm/clause" @@ -35,7 +36,7 @@ func (rf *RelationshipFilter) Create(m map[string]interface{}) { } func (rf *RelationshipFilter) GetById(db *database.Handler) (entity.Entity, error) { - r := &v1alpha2.RelationshipDefinition{} + r := &relationship.RelationshipDefinition{} err := db.First(r, "id = ?", rf.Id).Error if err != nil { return nil, registry.ErrGetById(err, rf.Id) diff --git a/models/meshmodel/registry/v1beta1/category_filter.go b/models/meshmodel/registry/v1beta1/category_filter.go index a2e17efa..399385a1 100644 --- a/models/meshmodel/registry/v1beta1/category_filter.go +++ b/models/meshmodel/registry/v1beta1/category_filter.go @@ -3,12 +3,13 @@ package v1beta1 import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/meshery/schemas/models/v1beta1/category" "gorm.io/gorm/clause" ) type CategoryFilter struct { - Id string + Id string Name string OrderOn string Greedy bool @@ -26,12 +27,12 @@ func (cf *CategoryFilter) Create(m map[string]interface{}) { } func (cf *CategoryFilter) GetById(db *database.Handler) (entity.Entity, error) { - c := &v1beta1.Category{} - err := db.First(c, "id = ?", cf.Id).Error + c := &category.CategoryDefinition{} + err := db.First(c, "id = ?", cf.Id).Error if err != nil { return nil, registry.ErrGetById(err, cf.Id) } - return c, err + return c, err } func (cf *CategoryFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index 12eb35ce..3265abfc 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -3,7 +3,6 @@ package v1beta1 import ( "fmt" - "github.com/MUzairS15/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/models/meshmodel/registry" @@ -40,7 +39,7 @@ type componentDefinitionWithModel struct { } func (cf *ComponentFilter) GetById(db *database.Handler) (entity.Entity, error) { - c := &v1beta1.ComponentDefinition{} + c := &component.ComponentDefinition{} err := db.First(c, "id = ?", cf.Id).Error if err != nil { return nil, registry.ErrGetById(err, cf.Id) diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index cc45197e..9bf77f56 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -3,10 +3,10 @@ package v1beta1 import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/meshery/schemas/models/v1alpha3/relationship" "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/model" - v1beta1 "github.com/meshery/schemas/models/v1beta1/model" "gorm.io/gorm/clause" ) @@ -39,7 +39,7 @@ func (mf *ModelFilter) Create(m map[string]interface{}) { mf.Name = m["name"].(string) } -func countUniqueModels(models []v1beta1.ModelDefinition) int { +func countUniqueModels(models []model.ModelDefinition) int { set := make(map[string]struct{}) for _, model := range models { key := model.Name + "@" + model.Model.Version @@ -51,7 +51,7 @@ func countUniqueModels(models []v1beta1.ModelDefinition) int { } func (mf *ModelFilter) GetById(db *database.Handler) (entity.Entity, error) { - m := &v1beta1.Model{} + m := &model.ModelDefinition{} // Retrieve the model by ID err := db.First(m, "id = ?", mf.Id).Error @@ -61,10 +61,10 @@ func (mf *ModelFilter) GetById(db *database.Handler) (entity.Entity, error) { // Include components if requested if mf.Components { - var components []v1beta1.ComponentDefinition - componentFinder := db.Model(&v1beta1.ComponentDefinition{}). + var components []component.ComponentDefinition + componentFinder := db.Model(&component.ComponentDefinition{}). Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version"). - Where("component_definition_dbs.model_id = ?", m.ID) + Where("component_definition_dbs.model_id = ?", m.Id) if err := componentFinder.Scan(&components).Error; err != nil { return nil, err } @@ -73,10 +73,10 @@ func (mf *ModelFilter) GetById(db *database.Handler) (entity.Entity, error) { // Include relationships if requested if mf.Relationships { - var relationships []v1alpha2.RelationshipDefinition - relationshipFinder := db.Model(&v1alpha2.RelationshipDefinition{}). + var relationships []relationship.RelationshipDefinition + relationshipFinder := db.Model(&relationship.RelationshipDefinition{}). Select("relationship_definition_dbs.*"). - Where("relationship_definition_dbs.model_id = ?", m.ID) + Where("relationship_definition_dbs.model_id = ?", m.Id) if err := relationshipFinder.Scan(&relationships).Error; err != nil { return nil, err } @@ -178,10 +178,10 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e // resolve for loop scope _modelDB := modelDB if includeComponents { - var components []v1beta1.ComponentDefinition - finder := db.Model(&v1beta1.ComponentDefinition{}). + var components []component.ComponentDefinition + finder := db.Model(&component.ComponentDefinition{}). Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version"). - Where("component_definition_dbs.model_id = ?", _modelDB.ID) + Where("component_definition_dbs.model_id = ?", _modelDB.Id) if err := finder.Scan(&components).Error; err != nil { return nil, 0, 0, err } diff --git a/models/registration/dir.go b/models/registration/dir.go index 2272e8ec..dc5167c0 100644 --- a/models/registration/dir.go +++ b/models/registration/dir.go @@ -6,10 +6,11 @@ import ( "path/filepath" "reflect" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/utils" + "github.com/meshery/schemas/models/v1alpha3/relationship" + "github.com/meshery/schemas/models/v1beta1/component" + "github.com/meshery/schemas/models/v1beta1/model" ) type Dir struct { @@ -17,35 +18,34 @@ type Dir struct { } /* - The directory should contain one and only one `model`. - A directory containing multiple `model` will be invalid. +The directory should contain one and only one `model`. +A directory containing multiple `model` will be invalid. */ func NewDir(path string) Dir { return Dir{dirpath: path} } - -/* - PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the packagingUnit struct. - Invalid definitions are stored in the regErrStore with error data. +/* +PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the packagingUnit struct. +Invalid definitions are stored in the regErrStore with error data. */ -func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err error){ +func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err error) { pkg := packagingUnit{} // check if the given is a directory _, err = os.ReadDir(d.dirpath) - if(err != nil){ + if err != nil { return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not read the directory: %e", err)) } - err = filepath.Walk(d.dirpath, func (path string, f os.FileInfo, err error) error { + err = filepath.Walk(d.dirpath, func(path string, f os.FileInfo, err error) error { if err != nil { return err } - if(f.IsDir()){ - return nil + if f.IsDir() { + return nil } byt, _ := os.ReadFile(path) - if(byt == nil){ - return nil + if byt == nil { + return nil } var e entity.Entity @@ -56,37 +56,37 @@ func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err e } // set it to pkgunit switch e.Type() { - case entity.Model: - if !reflect.ValueOf(pkg.model).IsZero() { - // currently models inside models are not handled - return nil - } - model, err := utils.Cast[*v1beta1.Model](e) - if(err != nil){ - regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) - } - pkg.model = *model - case entity.ComponentDefinition: - comp , err := utils.Cast[*v1beta1.ComponentDefinition](e) - if(err != nil){ - regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) - } - pkg.components = append(pkg.components, *comp) - case entity.RelationshipDefinition: - rel, err := utils.Cast[*v1alpha2.RelationshipDefinition](e) - if(err != nil){ - regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) - } - pkg.relationships = append(pkg.relationships, *rel) + case entity.Model: + if !reflect.ValueOf(pkg.model).IsZero() { + // currently models inside models are not handled + return nil + } + model, err := utils.Cast[*model.ModelDefinition](e) + if err != nil { + regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) + } + pkg.model = *model + case entity.ComponentDefinition: + comp, err := utils.Cast[*component.ComponentDefinition](e) + if err != nil { + regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) + } + pkg.components = append(pkg.components, *comp) + case entity.RelationshipDefinition: + rel, err := utils.Cast[*relationship.RelationshipDefinition](e) + if err != nil { + regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) + } + pkg.relationships = append(pkg.relationships, *rel) } return nil }) if err != nil { return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not completely walk the file tree: %e", err)) } - if (reflect.ValueOf(pkg.model).IsZero()){ + if reflect.ValueOf(pkg.model).IsZero() { err := fmt.Errorf("Model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core....... ...... must be present, otherwise the model package is considered malformed..") - regErrStore.AddInvalidDefinition(d.dirpath, err) + regErrStore.AddInvalidDefinition(d.dirpath, err) return pkg, err } return pkg, err diff --git a/models/registration/register.go b/models/registration/register.go index 8f35e8ba..b95e815e 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -1,17 +1,20 @@ package registration import ( - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" meshmodel "github.com/layer5io/meshkit/models/meshmodel/registry" + "github.com/meshery/schemas/models/v1alpha3/relationship" + "github.com/meshery/schemas/models/v1beta1/component" + "github.com/meshery/schemas/models/v1beta1/connection" + "github.com/meshery/schemas/models/v1beta1/model" ) // packaingUnit is the representation of the atomic unit that can be registered into the capabilities registry type packagingUnit struct { - model v1beta1.Model - components []v1beta1.ComponentDefinition - relationships []v1alpha2.RelationshipDefinition + model model.ModelDefinition + components []component.ComponentDefinition + relationships []relationship.RelationshipDefinition _ []v1beta1.PolicyDefinition } @@ -47,32 +50,54 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { model := pkg.model // Dont register anything else if registrant is not there - if model.Registrant.Hostname == "" { + if model.Registrant.Kind == "" { err := ErrMissingRegistrant(model.Name) - rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "", entity.Model, model.Name, err) + rh.regErrStore.InsertEntityRegError(model.Registrant.Kind, "", entity.Model, model.Name, err) return } - WriteAndReplaceSVGWithFileSystemPath(model.Metadata, rh.svgBaseDir, model.Name, model.Name) //Write SVG for models + if model.Metadata != nil { + svgComplete := "" + if model.Metadata.SvgComplete != nil { + svgComplete = *model.Metadata.SvgComplete + } + + //Write SVG for models + WriteAndReplaceSVGWithFileSystemPath(model.Metadata.SvgColor, + model.Metadata.SvgWhite, + svgComplete, rh.svgBaseDir, + model.Name, + model.Name, + ) + } _, _, err := rh.regManager.RegisterEntity( - v1beta1.Host{Hostname: model.Registrant.Hostname}, + connection.Connection{Kind: model.Registrant.Kind}, &model, ) // If model cannot be registered, don't register anything else if err != nil { err = ErrRegisterEntity(err, string(model.Type()), model.DisplayName) - rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "", entity.Model, model.Name, err) + rh.regErrStore.InsertEntityRegError(model.Registrant.Kind, "", entity.Model, model.Name, err) return } - hostname := model.Registrant.Hostname + hostname := model.Registrant.Kind modelName := model.Name // 2. Register components for _, comp := range pkg.components { comp.Model = model - WriteAndReplaceSVGWithFileSystemPath(comp.Metadata, rh.svgBaseDir, comp.Model.Name, comp.Component.Kind) //Write SVG on components + if comp.Styles != nil { + //Write SVG for components + WriteAndReplaceSVGWithFileSystemPath( + comp.Styles.SvgColor, + comp.Styles.SvgWhite, + comp.Styles.SvgComplete, + rh.svgBaseDir, + comp.Model.Name, + comp.Component.Kind) //Write SVG on components + } _, _, err := rh.regManager.RegisterEntity( - v1beta1.Host{Hostname: hostname}, + connection.Connection{Kind: model.Registrant.Kind}, &comp, ) if err != nil { @@ -84,12 +109,10 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { // 3. Register relationships for _, rel := range pkg.relationships { rel.Model = model - _, _, err := rh.regManager.RegisterEntity(v1beta1.Host{ - Hostname: hostname, - }, &rel) + _, _, err := rh.regManager.RegisterEntity(connection.Connection{Kind: model.Registrant.Kind}, &rel) if err != nil { - err = ErrRegisterEntity(err, string(rel.Type()), rel.Kind) - rh.regErrStore.InsertEntityRegError(hostname, modelName, entity.RelationshipDefinition, rel.ID.String(), err) + err = ErrRegisterEntity(err, string(rel.Type()), string(rel.Kind)) + rh.regErrStore.InsertEntityRegError(hostname, modelName, entity.RelationshipDefinition, rel.Id.String(), err) } } } diff --git a/models/registration/svg_helper.go b/models/registration/svg_helper.go index 88f0b35a..086e5673 100644 --- a/models/registration/svg_helper.go +++ b/models/registration/svg_helper.go @@ -20,7 +20,7 @@ func writeHashCheckSVG(key string, val string) { mx.Unlock() } -func WriteAndReplaceSVGWithFileSystemPath(metadata map[string]interface{}, baseDir, dirname, filename string) { +func WriteAndReplaceSVGWithFileSystemPath(svgColor, svgWhite, svgComplete string, baseDir, dirname, filename string) (svgColorPath, svgWhitePath, svgCompletePath string) { filename = strings.ToLower(filename) successCreatingDirectory := false defer func() { @@ -28,7 +28,7 @@ func WriteAndReplaceSVGWithFileSystemPath(metadata map[string]interface{}, baseD UISVGPaths = append(UISVGPaths, filepath.Join(baseDir, dirname)) } }() - if metadata["svgColor"] != "" { + if svgColor != "" { path := filepath.Join(baseDir, dirname, "color") err := os.MkdirAll(path, 0777) if err != nil { @@ -37,31 +37,29 @@ func WriteAndReplaceSVGWithFileSystemPath(metadata map[string]interface{}, baseD } successCreatingDirectory = true - x, ok := metadata["svgColor"].(string) - if ok { - hash := md5.Sum([]byte(x)) - hashString := hex.EncodeToString(hash[:]) - pathsvg := hashCheckSVG[hashString] - if pathsvg != "" { // the image has already been loaded, point the component to that path - metadata["svgColor"] = pathsvg - goto White - } - f, err := os.Create(filepath.Join(path, filename+"-color.svg")) - if err != nil { - fmt.Println(err) - return - } - _, err = f.WriteString(x) - if err != nil { - fmt.Println(err) - return - } - metadata["svgColor"] = getRelativePathForAPI(baseDir, filepath.Join(dirname, "color", filename+"-color.svg")) //Replace the actual SVG with path to SVG - writeHashCheckSVG(hashString, metadata["svgColor"].(string)) + hash := md5.Sum([]byte(svgColor)) + hashString := hex.EncodeToString(hash[:]) + pathsvg := hashCheckSVG[hashString] + if pathsvg != "" { // the image has already been loaded, point the component to that path + svgColorPath = pathsvg + goto White } + f, err := os.Create(filepath.Join(path, filename+"-color.svg")) + if err != nil { + fmt.Println(err) + return + } + _, err = f.WriteString(svgColor) + if err != nil { + fmt.Println(err) + return + } + svgColorPath = getRelativePathForAPI(baseDir, filepath.Join(dirname, "color", filename+"-color.svg")) //Replace the actual SVG with path to SVG + writeHashCheckSVG(hashString, svgColor) + } White: - if metadata["svgWhite"] != "" { + if svgWhite != "" { path := filepath.Join(baseDir, dirname, "white") err := os.MkdirAll(path, 0777) if err != nil { @@ -70,31 +68,29 @@ White: } successCreatingDirectory = true - x, ok := metadata["svgWhite"].(string) - if ok { - hash := md5.Sum([]byte(x)) - hashString := hex.EncodeToString(hash[:]) - pathsvg := hashCheckSVG[hashString] - if pathsvg != "" { // the image has already been loaded, point the component to that path - metadata["svgWhite"] = pathsvg - goto Complete - } - f, err := os.Create(filepath.Join(path, filename+"-white.svg")) - if err != nil { - fmt.Println(err) - return - } - _, err = f.WriteString(x) - if err != nil { - fmt.Println(err) - return - } - metadata["svgWhite"] = getRelativePathForAPI(baseDir, filepath.Join(dirname, "white", filename+"-white.svg")) //Replace the actual SVG with path to SVG - writeHashCheckSVG(hashString, metadata["svgWhite"].(string)) + hash := md5.Sum([]byte(svgWhite)) + hashString := hex.EncodeToString(hash[:]) + pathsvg := hashCheckSVG[hashString] + if pathsvg != "" { // the image has already been loaded, point the component to that path + svgWhitePath = pathsvg + goto Complete } + f, err := os.Create(filepath.Join(path, filename+"-white.svg")) + if err != nil { + fmt.Println(err) + return + } + _, err = f.WriteString(svgWhite) + if err != nil { + fmt.Println(err) + return + } + svgWhitePath = getRelativePathForAPI(baseDir, filepath.Join(dirname, "white", filename+"-white.svg")) //Replace the actual SVG with path to SVG + writeHashCheckSVG(hashString, svgWhite) + } Complete: - if metadata["svgComplete"] != "" { + if svgComplete != "" { path := filepath.Join(baseDir, dirname, "complete") err := os.MkdirAll(path, 0777) if err != nil { @@ -103,31 +99,32 @@ Complete: } successCreatingDirectory = true - x, ok := metadata["svgComplete"].(string) - if ok { - hash := md5.Sum([]byte(x)) - hashString := hex.EncodeToString(hash[:]) - pathsvg := hashCheckSVG[hashString] - if pathsvg != "" { // the image has already been loaded, point the component to that path - metadata["svgComplete"] = pathsvg - return - } - f, err := os.Create(filepath.Join(path, filename+"-complete.svg")) - if err != nil { - fmt.Println(err) - return - } - _, err = f.WriteString(x) - if err != nil { - fmt.Println(err) - return - } - metadata["svgComplete"] = getRelativePathForAPI(baseDir, filepath.Join(dirname, "complete", filename+"-complete.svg")) //Replace the actual SVG with path to SVG - writeHashCheckSVG(hashString, metadata["svgComplete"].(string)) + hash := md5.Sum([]byte(svgComplete)) + hashString := hex.EncodeToString(hash[:]) + pathsvg := hashCheckSVG[hashString] + if pathsvg != "" { // the image has already been loaded, point the component to that path + svgCompletePath = pathsvg + return + } + f, err := os.Create(filepath.Join(path, filename+"-complete.svg")) + if err != nil { + fmt.Println(err) + return } + _, err = f.WriteString(svgComplete) + if err != nil { + fmt.Println(err) + return + } + svgCompletePath = getRelativePathForAPI(baseDir, filepath.Join(dirname, "complete", filename+"-complete.svg")) //Replace the actual SVG with path to SVG + writeHashCheckSVG(hashString, svgComplete) + } + return } +// func WriteAndReplaceSVGWithFileSystemPath(metadata map[string]interface{}, baseDir, dirname, filename string) { + func getRelativePathForAPI(baseDir, path string) string { ui := strings.TrimPrefix(baseDir, "../../") return filepath.Join(ui, path) diff --git a/models/registration/utils.go b/models/registration/utils.go index 9c96de1a..354c2fb5 100644 --- a/models/registration/utils.go +++ b/models/registration/utils.go @@ -4,17 +4,20 @@ import ( "encoding/json" "fmt" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/meshery/schemas/models/v1alpha3" + "github.com/meshery/schemas/models/v1alpha3/relationship" + "github.com/meshery/schemas/models/v1beta1" + "github.com/meshery/schemas/models/v1beta1/component" + "github.com/meshery/schemas/models/v1beta1/model" "gopkg.in/yaml.v2" ) func unmarshal(byt []byte, out interface{}) error { err := json.Unmarshal(byt, out) - if(err != nil){ + if err != nil { err = yaml.Unmarshal(byt, out) - if(err != nil){ + if err != nil { return fmt.Errorf("Not a valid YAML or JSON") } } @@ -23,35 +26,35 @@ func unmarshal(byt []byte, out interface{}) error { // TODO: refactor this and use CUE func getEntity(byt []byte) (et entity.Entity, _ error) { - var versionMeta v1beta1.VersionMeta - err := unmarshal(byt, &versionMeta) - if err != nil || versionMeta.SchemaVersion == "" { + var schemaVersion string + err := unmarshal(byt, &schemaVersion) + if err != nil || schemaVersion == "" { return nil, ErrGetEntity(fmt.Errorf("Does not contain versionmeta")) } - switch (versionMeta.SchemaVersion) { - case v1beta1.ComponentSchemaVersion: - var compDef v1beta1.ComponentDefinition - err := unmarshal(byt, &compDef) - if err != nil { - return nil, ErrGetEntity(fmt.Errorf("Invalid component definition: %s", err.Error())) - } - et = &compDef - case v1beta1.ModelSchemaVersion: - var model v1beta1.Model - err := unmarshal(byt,&model) - if err != nil { - return nil, ErrGetEntity(fmt.Errorf("Invalid model definition: %s", err.Error())) - } - et = &model - case v1alpha2.RelationshipSchemaVersion: - var rel v1alpha2.RelationshipDefinition - err := unmarshal(byt,&rel) - if err != nil { - return nil, ErrGetEntity(fmt.Errorf("Invalid relationship definition: %s", err.Error())) - } - et = &rel - default: - return nil, ErrGetEntity(fmt.Errorf("Not a valid component definition, model definition, or relationship definition")) + switch schemaVersion { + case v1beta1.ComponentSchemaVersion: + var compDef component.ComponentDefinition + err := unmarshal(byt, &compDef) + if err != nil { + return nil, ErrGetEntity(fmt.Errorf("Invalid component definition: %s", err.Error())) + } + et = &compDef + case v1beta1.ModelSchemaVersion: + var model model.ModelDefinition + err := unmarshal(byt, &model) + if err != nil { + return nil, ErrGetEntity(fmt.Errorf("Invalid model definition: %s", err.Error())) + } + et = &model + case v1alpha3.RelationshipSchemaVersion: + var rel relationship.RelationshipDefinition + err := unmarshal(byt, &rel) + if err != nil { + return nil, ErrGetEntity(fmt.Errorf("Invalid relationship definition: %s", err.Error())) + } + et = &rel + default: + return nil, ErrGetEntity(fmt.Errorf("Not a valid component definition, model definition, or relationship definition")) } return et, nil } diff --git a/utils/cue.go b/utils/cue.go index 68a6b036..229edfb7 100644 --- a/utils/cue.go +++ b/utils/cue.go @@ -91,7 +91,6 @@ func JsonSchemaToCue(value string) (cue.Value, error) { PkgName: "jsonschemeconv", }) - s := errors.Details(err, nil) if err != nil { return out, ErrJsonSchemaToCue(err) } diff --git a/validator/validate.go b/validator/validate.go index 024a2ef1..dc043d52 100644 --- a/validator/validate.go +++ b/validator/validate.go @@ -2,7 +2,6 @@ package validator import ( "encoding/json" - "fmt" "sync" "cuelang.org/go/cue" @@ -62,7 +61,6 @@ func GetSchemaFor(resourceName string) (cue.Value, error) { return schema, utils.ErrMarshal(err) } - schema, err = utils.JsonSchemaToCue(string(byt)) log, _ := logger.New("test-validate", logger.Options{}) @@ -88,7 +86,7 @@ func Validate(schema cue.Value, resourceValue interface{}) error { valid, errs := utils.Validate(schema, cv) if !valid { - errs := convertCueErrorsToStrings(errs) + errs := convertCueErrorsToStrings(errs) return errors.New(ErrValidateCode, errors.Alert, []string{"validation for the resource failed"}, @@ -106,7 +104,7 @@ func convertCueErrorsToStrings(errs []cueerrors.Error) []string { _ = cueerrors.Sanitize(err) } for _, err2 := range errs { - + res = append(res, err2.Error()) } return res From fc9b221b9a91625eb597d6c6262e4941ab3622b2 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 12 Aug 2024 22:18:29 +0530 Subject: [PATCH 026/112] update Signed-off-by: MUzairS15 --- models/meshmodel/core/policies/data_models.go | 3 --- .../core/policies/rego_policy_relationship.go | 14 ++++---------- 2 files changed, 4 insertions(+), 13 deletions(-) delete mode 100644 models/meshmodel/core/policies/data_models.go diff --git a/models/meshmodel/core/policies/data_models.go b/models/meshmodel/core/policies/data_models.go deleted file mode 100644 index 4f7f9c04..00000000 --- a/models/meshmodel/core/policies/data_models.go +++ /dev/null @@ -1,3 +0,0 @@ -package policies - -// Add response struct based on schema for all relationships evaluations. binding, network, hierarchical diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index 88fa8951..7726f2dc 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -6,12 +6,12 @@ import ( "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/layer5io/meshkit/models/meshmodel/registry/v1alpha3" - "github.com/layer5io/meshkit/utils" + "github.com/meshery/schemas/models/v1beta1/pattern" "github.com/open-policy-agent/opa/rego" + "github.com/open-policy-agent/opa/storage" "github.com/open-policy-agent/opa/storage/inmem" "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" ) type Rego struct { @@ -48,7 +48,7 @@ func NewRegoInstance(policyDir string, regManager *registry.RegistryManager) (*R } // RegoPolicyHandler takes the required inputs and run the query against all the policy files provided -func (r *Rego) RegoPolicyHandler(regoQueryString string, designFile []byte) (interface{}, error) { +func (r *Rego) RegoPolicyHandler(designFile pattern.PatternFile, regoQueryString string, relationshipsToEvalaute ...string) (interface{}, error) { if r == nil { return nil, ErrEval(fmt.Errorf("policy engine is not yet ready")) } @@ -63,13 +63,7 @@ func (r *Rego) RegoPolicyHandler(regoQueryString string, designFile []byte) (int return nil, ErrPrepareForEval(err) } - var input map[string]interface{} - err = yaml.Unmarshal((designFile), &input) - if err != nil { - return nil, utils.ErrUnmarshal(err) - } - - eval_result, err := regoEngine.Eval(r.ctx, rego.EvalInput(input)) + eval_result, err := regoEngine.Eval(r.ctx, rego.EvalInput(designFile)) if err != nil { return nil, ErrEval(err) } From c88db2b4f819ed4790926da4e6734e7afa51a57d Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 13 Aug 2024 12:17:30 +0530 Subject: [PATCH 027/112] update getEntity func Signed-off-by: MUzairS15 --- models/registration/utils.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/models/registration/utils.go b/models/registration/utils.go index 354c2fb5..d3699fac 100644 --- a/models/registration/utils.go +++ b/models/registration/utils.go @@ -26,12 +26,15 @@ func unmarshal(byt []byte, out interface{}) error { // TODO: refactor this and use CUE func getEntity(byt []byte) (et entity.Entity, _ error) { - var schemaVersion string - err := unmarshal(byt, &schemaVersion) - if err != nil || schemaVersion == "" { + type schemaVersion struct { + SchemaVersion string `json:"schemaVersion" yaml:"schemaVersion"` + } + var sv schemaVersion + err := unmarshal(byt, &sv) + if err != nil || sv.SchemaVersion == "" { return nil, ErrGetEntity(fmt.Errorf("Does not contain versionmeta")) } - switch schemaVersion { + switch sv.SchemaVersion { case v1beta1.ComponentSchemaVersion: var compDef component.ComponentDefinition err := unmarshal(byt, &compDef) From 11081d01041aa4bc8db12382faa02ccf2d4fec90 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 13 Aug 2024 15:09:15 +0530 Subject: [PATCH 028/112] remove redundant logic Signed-off-by: MUzairS15 --- go.mod | 69 +++---- go.sum | 190 +++++++++--------- .../core/policies/rego_policy_relationship.go | 32 ++- 3 files changed, 151 insertions(+), 140 deletions(-) diff --git a/go.mod b/go.mod index 7b0ba119..fd79534c 100644 --- a/go.mod +++ b/go.mod @@ -18,23 +18,23 @@ require ( github.com/fluxcd/pkg/oci v0.34.0 github.com/fluxcd/pkg/tar v0.4.0 github.com/go-git/go-git/v5 v5.11.0 - github.com/go-logr/logr v1.3.0 + github.com/go-logr/logr v1.4.2 github.com/gofrs/uuid v4.4.0+incompatible github.com/google/go-containerregistry v0.17.0 - github.com/google/uuid v1.5.0 + github.com/google/uuid v1.6.0 github.com/kubernetes/kompose v1.31.1 github.com/layer5io/meshery-operator v0.7.0 github.com/meshery/schemas v0.7.9 github.com/nats-io/nats.go v1.31.0 - github.com/open-policy-agent/opa v0.57.1 - github.com/opencontainers/image-spec v1.1.0-rc6 + github.com/open-policy-agent/opa v0.67.1 + github.com/opencontainers/image-spec v1.1.0 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cobra v1.8.0 - github.com/spf13/viper v1.17.0 - golang.org/x/oauth2 v0.15.0 - golang.org/x/text v0.14.0 - google.golang.org/api v0.152.0 + github.com/spf13/cobra v1.8.1 + github.com/spf13/viper v1.18.2 + golang.org/x/oauth2 v0.20.0 + golang.org/x/text v0.16.0 + google.golang.org/api v0.153.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/postgres v1.5.3 @@ -50,8 +50,7 @@ require ( ) require ( - cloud.google.com/go/compute v1.23.3 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect dario.cat/mergo v1.0.0 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 // indirect @@ -64,8 +63,7 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Masterminds/squirrel v1.5.4 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/Microsoft/hcsshim v0.11.4 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect @@ -85,12 +83,14 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect github.com/aws/smithy-go v1.19.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/cockroachdb/apd/v3 v3.2.1 // indirect - github.com/containerd/containerd v1.7.11 // indirect + github.com/containerd/containerd v1.7.20 // indirect + github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -129,7 +129,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect @@ -172,7 +172,6 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mattn/go-sqlite3 v1.14.17 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect @@ -200,9 +199,9 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/rubenv/sql-migrate v1.5.2 // indirect @@ -226,26 +225,24 @@ require ( github.com/xlab/treeprint v1.2.0 // indirect github.com/yashtewari/glob-intersection v0.2.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect - go.opentelemetry.io/otel v1.21.0 // indirect - go.opentelemetry.io/otel/metric v1.21.0 // indirect - go.opentelemetry.io/otel/sdk v1.21.0 // indirect - go.opentelemetry.io/otel/trace v1.21.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.25.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/term v0.22.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.16.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect - google.golang.org/grpc v1.60.1 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/grpc v1.65.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 356c54af..58dff424 100644 --- a/go.sum +++ b/go.sum @@ -7,10 +7,8 @@ cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxK cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= -cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -55,12 +53,12 @@ github.com/Microsoft/go-winio v0.3.8/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyv github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/go-winio v0.4.15-0.20200113171025-3fe6c5262873/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= -github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= -github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= +github.com/Microsoft/hcsshim v0.11.7 h1:vl/nj3Bar/CvJSYo7gIQPyRWc9f3c6IeSNavBTSZNZQ= +github.com/Microsoft/hcsshim v0.11.7/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= @@ -136,13 +134,13 @@ github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3k github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytecodealliance/wasmtime-go/v3 v3.0.2 h1:3uZCA/BLTIu+DqCfguByNMJa2HVHpXvjfy0Dy7g6fuA= github.com/bytecodealliance/wasmtime-go/v3 v3.0.2/go.mod h1:RnUjnIXxEJcL6BgCvNyzCCRzZcxCgsZCi+RNlvYor5Q= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -160,16 +158,20 @@ github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHq github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw= -github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE= +github.com/containerd/containerd v1.7.20 h1:Sl6jQYk3TRavaU83h66QMbI2Nqg9Jm6qzwX57Vsn1SQ= +github.com/containerd/containerd v1.7.20/go.mod h1:52GsS5CwquuqPuLncsXwG0t2CiUce+KsNHJZQJvAgR0= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20200228182428-0f16d7a0959c/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY= github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= +github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= @@ -183,7 +185,7 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -280,8 +282,8 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjr github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= -github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4= +github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= +github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -317,8 +319,9 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= @@ -365,8 +368,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= -github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= +github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= +github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -389,8 +392,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= @@ -432,8 +435,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -460,8 +463,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmg github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -603,13 +606,11 @@ github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/meshery/kompose v1.0.1 h1:lg8B/pkLh6762jeFsQATD8UJZZwXZf/aviC3/dzw78A= github.com/meshery/kompose v1.0.1/go.mod h1:TWhWTEMbJBUzENf4JTEtBmZRFm/r8n0nS6v4/nSD2vA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= -github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= +github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -690,8 +691,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= -github.com/open-policy-agent/opa v0.57.1 h1:LAa4Z0UkpjV94nRLy6XCvgOacQ6N1jf8TJLMUIzFRqc= -github.com/open-policy-agent/opa v0.57.1/go.mod h1:YYcVsWcdOW47owR0zElx8HPYZK60vL0MOPsEmh13us4= +github.com/open-policy-agent/opa v0.67.1 h1:rzy26J6g1X+CKknAcx0Vfbt41KqjuSzx4E0A8DAZf3E= +github.com/open-policy-agent/opa v0.67.1/go.mod h1:aqKlHc8E2VAAylYE9x09zJYr/fYzGX+JKne89UGqFzk= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -699,8 +700,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v0.0.0-20170515205857-f03dbe35d449/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.1.0-rc6 h1:XDqvyKsJEbRtATzkgItUqBA7QHk58yxX1Ov9HERHNqU= -github.com/opencontainers/image-spec v1.1.0-rc6/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runc v0.0.0-20161109192122-51371867a01c/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -738,19 +739,19 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -765,8 +766,8 @@ github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0= github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -810,8 +811,8 @@ github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -820,8 +821,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI= -github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -837,8 +838,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= @@ -878,22 +880,22 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= -go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= -go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= -go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= -go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= -go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= -go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= -go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= -go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= -go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -916,8 +918,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -943,8 +945,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -976,13 +978,13 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= -golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -993,8 +995,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1040,16 +1042,16 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -1057,12 +1059,11 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= @@ -1096,8 +1097,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1109,14 +1110,12 @@ google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.152.0 h1:t0r1vPnfMc260S2Ci+en7kfCZaLOPs5KI0sVV/6jZrY= -google.golang.org/api v0.152.0/go.mod h1:3qNJX5eOmhiWYc67jRA/3GsDw97UFb5ivv7Y2PrriAY= +google.golang.org/api v0.153.0 h1:N1AwGhielyKFaUqH07/ZSIQR3uNPcV7NVw0vj+j4iR4= +google.golang.org/api v0.153.0/go.mod h1:3qNJX5eOmhiWYc67jRA/3GsDw97UFb5ivv7Y2PrriAY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1128,12 +1127,11 @@ google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBr google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= -google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= -google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= -google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc= +google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 h1:1hfbdAfFbkmpg41000wDVqr7jUpK/Yo+LPnIxxGzmkg= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1144,8 +1142,8 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= -google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1159,8 +1157,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index 7726f2dc..4f26ddca 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -6,6 +6,7 @@ import ( "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/layer5io/meshkit/models/meshmodel/registry/v1alpha3" + "github.com/layer5io/meshkit/utils" "github.com/meshery/schemas/models/v1beta1/pattern" "github.com/open-policy-agent/opa/rego" @@ -67,15 +68,30 @@ func (r *Rego) RegoPolicyHandler(designFile pattern.PatternFile, regoQueryString if err != nil { return nil, ErrEval(err) } - if !eval_result.Allowed() { - if len(eval_result) > 0 { - if len(eval_result[0].Expressions) > 0 { - return eval_result[0].Expressions[0].Value, nil - } - return nil, ErrEval(fmt.Errorf("evaluation results are empty")) - } + + if len(eval_result) == 0 { + return nil, ErrEval(fmt.Errorf("evaluation results are empty")) + } + + if len(eval_result[0].Expressions) == 0 { return nil, ErrEval(fmt.Errorf("evaluation results are empty")) } - return nil, ErrEval(err) + result, err := utils.Cast[map[string]interface{}](eval_result[0].Expressions[0].Value) + if err != nil { + return nil, ErrEval(err) + } + + evalResults, ok := result["evaluate"] + if !ok { + return nil, ErrEval(fmt.Errorf("evaluation results are empty")) + } + + evaluatedPatternFile, err := utils.MarshalAndUnmarshal[interface{}, pattern.PatternFile](evalResults) + if err != nil { + return nil, err + } + + return evaluatedPatternFile, nil + } From a091e94f9acfba005b130b941ad78cc06b5e3854 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 13 Aug 2024 16:04:34 +0530 Subject: [PATCH 029/112] update db queries Signed-off-by: MUzairS15 --- .../registry/v1alpha3/relationship_filter.go | 1 - .../meshmodel/registry/v1beta1/model_filter.go | 16 ++++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/models/meshmodel/registry/v1alpha3/relationship_filter.go b/models/meshmodel/registry/v1alpha3/relationship_filter.go index e3893ce9..624c683a 100644 --- a/models/meshmodel/registry/v1alpha3/relationship_filter.go +++ b/models/meshmodel/registry/v1alpha3/relationship_filter.go @@ -51,7 +51,6 @@ func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entit Joins("JOIN model_dbs ON relationship_definition_dbs.model_id = model_dbs.id"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") - // TODO(@MUzairS15): Refactor this once Status is made a first class field in RelationshipFilter status := "enabled" if relationshipFilter.Status != "" { diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index 9bf77f56..1819f1d2 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -146,6 +146,14 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e finder = finder.Order("display_name") } + status := "enabled" + + if mf.Status != "" { + status = mf.Status + } + + finder = finder.Where("model_dbs.status = ?", status) + finder.Count(&count) if mf.Limit != 0 { @@ -155,14 +163,6 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e finder = finder.Offset(mf.Offset) } - status := "enabled" - - if mf.Status != "" { - status = mf.Status - } - - finder = finder.Where("model_dbs.status = ?", status) - includeComponents = mf.Components includeRelationships = mf.Relationships From a7afe56d91607475cda8b716e795e241020941f1 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 13 Aug 2024 21:17:55 +0530 Subject: [PATCH 030/112] update schemas to v0.7.17 Signed-off-by: MUzairS15 --- go.mod | 4 +--- go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fd79534c..6c8609d1 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/google/uuid v1.6.0 github.com/kubernetes/kompose v1.31.1 github.com/layer5io/meshery-operator v0.7.0 - github.com/meshery/schemas v0.7.9 + github.com/meshery/schemas v0.7.17 github.com/nats-io/nats.go v1.31.0 github.com/open-policy-agent/opa v0.67.1 github.com/opencontainers/image-spec v1.1.0 @@ -260,5 +260,3 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) - -replace github.com/meshery/schemas => ../schemas diff --git a/go.sum b/go.sum index 58dff424..b6991647 100644 --- a/go.sum +++ b/go.sum @@ -608,6 +608,8 @@ github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/meshery/kompose v1.0.1 h1:lg8B/pkLh6762jeFsQATD8UJZZwXZf/aviC3/dzw78A= github.com/meshery/kompose v1.0.1/go.mod h1:TWhWTEMbJBUzENf4JTEtBmZRFm/r8n0nS6v4/nSD2vA= +github.com/meshery/schemas v0.7.17 h1:NOfPAECsaU0l5hASDMlu6yPBlwLkGn9CKUq9umjkCE0= +github.com/meshery/schemas v0.7.17/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= From 8adfef2390c9de68e1176306ce61e53a03baab0a Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 13 Aug 2024 22:43:55 +0530 Subject: [PATCH 031/112] run gofmt Signed-off-by: MUzairS15 --- generators/artifacthub/package.go | 4 +- generators/models/interfaces.go | 2 +- models/meshmodel/registry/error.go | 16 +- models/meshmodel/registry/registry.go | 2 +- .../registry/v1beta1/policy_filter.go | 8 +- models/registration/error.go | 168 +++++++++--------- models/registration/interface.go | 14 +- models/registration/oci.go | 8 +- models/registration/tar.go | 4 +- 9 files changed, 110 insertions(+), 116 deletions(-) diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index 4e1ef920..3c018813 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -10,8 +10,8 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" "github.com/layer5io/meshkit/utils/manifests" - _component "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/category" + _component "github.com/meshery/schemas/models/v1beta1/component" "gopkg.in/yaml.v2" ) @@ -53,7 +53,7 @@ func (pkg AhPackage) GenerateComponents() ([]_component.ComponentDefinition, err if err != nil { continue } - + if comp.Model.Metadata.AdditionalProperties == nil { comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) } diff --git a/generators/models/interfaces.go b/generators/models/interfaces.go index 6aa0c808..19f61177 100644 --- a/generators/models/interfaces.go +++ b/generators/models/interfaces.go @@ -1,6 +1,6 @@ package models -import "github.com/meshery/schemas/models/v1beta1/component" +import "github.com/meshery/schemas/models/v1beta1/component" // anything that can be validated is a Validator type Validator interface { diff --git a/models/meshmodel/registry/error.go b/models/meshmodel/registry/error.go index 2566ec58..2e0351b4 100644 --- a/models/meshmodel/registry/error.go +++ b/models/meshmodel/registry/error.go @@ -14,18 +14,18 @@ var ( ErrRegisteringEntityCode = "replace_me" ErrUnknownHostInMapCode = "replace_me" ErrCreatingUserDataDirectoryCode = "replace_me" - ErrGetByIdCode = "replace_me" + ErrGetByIdCode = "replace_me" ) func ErrGetById(err error, id string) error { return errors.New( - ErrUnknownHostCode, - errors.Alert, - []string{"Failed to get the entity with the given ID: " + id}, - []string{err.Error()}, - []string{"Entity with the given ID may not be present in the registry", "Registry might be inaccessible at the moment"}, - []string{"Check if your ID is correct" , "If the registry is inaccesible, please try again after some time"}, - ) + ErrUnknownHostCode, + errors.Alert, + []string{"Failed to get the entity with the given ID: " + id}, + []string{err.Error()}, + []string{"Entity with the given ID may not be present in the registry", "Registry might be inaccessible at the moment"}, + []string{"Check if your ID is correct", "If the registry is inaccesible, please try again after some time"}, + ) } diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index 476a9055..91606a61 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -199,7 +199,7 @@ func (rm *RegistryManager) GetEntities(f entity.Filter) ([]entity.Entity, int64, return f.Get(rm.db) } -func (rm *RegistryManager) GetEntityById (f entity.Filter) (entity.Entity, error) { +func (rm *RegistryManager) GetEntityById(f entity.Filter) (entity.Entity, error) { return f.GetById(rm.db) } diff --git a/models/meshmodel/registry/v1beta1/policy_filter.go b/models/meshmodel/registry/v1beta1/policy_filter.go index 7de9ce07..d15d970c 100644 --- a/models/meshmodel/registry/v1beta1/policy_filter.go +++ b/models/meshmodel/registry/v1beta1/policy_filter.go @@ -8,7 +8,7 @@ import ( ) type PolicyFilter struct { - Id string + Id string Kind string Greedy bool SubType string @@ -26,12 +26,12 @@ func (pf *PolicyFilter) Create(m map[string]interface{}) { } func (pf *PolicyFilter) GetById(db *database.Handler) (entity.Entity, error) { - p := &v1beta1.PolicyDefinition{} - err := db.First(p, "id = ?", pf.Id).Error + p := &v1beta1.PolicyDefinition{} + err := db.First(p, "id = ?", pf.Id).Error if err != nil { return nil, registry.ErrGetById(err, pf.Id) } - return p, err + return p, err } func (pf *PolicyFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { diff --git a/models/registration/error.go b/models/registration/error.go index 73af3795..32e627c3 100644 --- a/models/registration/error.go +++ b/models/registration/error.go @@ -1,85 +1,83 @@ -package registration - -import ( - "fmt" - - "github.com/layer5io/meshkit/errors" -) - -const ( - ErrDirPkgUnitParseFailCode = "replace_me" - ErrGetEntityCode = "replace_me" - ErrRegisterEntityCode = "replace_me" - ErrImportFailureCode = "replace_me" - ErrMissingRegistrantCode = "replace_me" - ErrSeedingComponentsCode = "replace-me" -) - - - -func ErrSeedingComponents(err error) error { - return errors.New( - ErrSeedingComponentsCode, - errors.Alert, - []string{"Failed to register the given models into meshery's registry"}, - []string{err.Error()}, - []string{"Given models may not be in accordance with Meshery's schema", "Internal(OS level) error while reading files" }, - []string{"Make sure the models being seeded are valid in accordance with Meshery's schema", "If it is an internal error, please try again after some time"}, - ) -} - -func ErrMissingRegistrant(modelName string) error { - return errors.New( - ErrMissingRegistrantCode, - errors.Alert, - []string{fmt.Sprintf("Model with name: %s does not have registrant information", modelName)}, - []string{"Meshery models are always registered in context of a registrant."}, - // there is only one cause for this error - []string{""}, - []string{"Make sure that the registrant information is present in the model definition"}, - ) -} - -func ErrRegisterEntity(err error, name, entity string) error { - return errors.New( - ErrRegisterEntityCode, - errors.Alert, - []string{fmt.Sprintf("Failed to register an entity of name: %s and type: %s into Meshery's registry", name, entity)}, - []string{err.Error()}, - []string{fmt.Sprintf("%s definition might be violating the definition schema", entity), fmt.Sprintf("%s might be missing model details", entity)}, - []string{fmt.Sprintf("ensure the %s definition follows the correct schema", entity), fmt.Sprintf("ensure %s definition belongs to correct model", entity)}, - ) -} - -func ErrGetEntity(err error) error { - return errors.New( - ErrGetEntityCode, - errors.Alert, - []string{"Could not parse the given data into any Meshery entity"}, - []string{err.Error()}, - []string{"Entity definition might not be in accordance with it's corresponding schema", "Might be an invalid/unsupported schemaVersion"}, - []string{"Ensure that the definition given is in accordance with it's schema", "Ensure that the schemaVersion used is valid and is supported by Meshery"}, - ) -} - -func ErrDirPkgUnitParseFail(dirpath string, err error) error { - return errors.New( - ErrDirPkgUnitParseFailCode, - errors.Alert, - []string{fmt.Sprintf("Directory at path: %s cannot be registered into Meshery", dirpath)}, - []string{fmt.Sprint(err.Error())}, - []string{"The directory might not have a valid model definition", "Might be some internal issues while walking the file tree"}, - []string{"Make sure that there is a valid model definition present in the directory. Meshery's registration pipeline currently does not support nested models, therefore the behaviour might be unexpected if it contains nested models.", "If there is an internal error, please try again after some time"}, - ) -} - -func ErrImportFailure(hostname string, failedMsg string) error { - return errors.New( - ErrImportFailureCode, - errors.Alert, - []string{fmt.Sprintf("Errors while registering entities for registrant: %s", hostname)}, - []string{failedMsg}, - []string{"Entity definition might not be in accordance with schema", "Entity version might not be supported by Meshery"}, - []string{"See the registration logs (found at $HOME/.meshery/logs/registry/registry-logs.log) to find out which Entity failed to be imported with more specific error information."}, - ) -} \ No newline at end of file +package registration + +import ( + "fmt" + + "github.com/layer5io/meshkit/errors" +) + +const ( + ErrDirPkgUnitParseFailCode = "replace_me" + ErrGetEntityCode = "replace_me" + ErrRegisterEntityCode = "replace_me" + ErrImportFailureCode = "replace_me" + ErrMissingRegistrantCode = "replace_me" + ErrSeedingComponentsCode = "replace-me" +) + +func ErrSeedingComponents(err error) error { + return errors.New( + ErrSeedingComponentsCode, + errors.Alert, + []string{"Failed to register the given models into meshery's registry"}, + []string{err.Error()}, + []string{"Given models may not be in accordance with Meshery's schema", "Internal(OS level) error while reading files"}, + []string{"Make sure the models being seeded are valid in accordance with Meshery's schema", "If it is an internal error, please try again after some time"}, + ) +} + +func ErrMissingRegistrant(modelName string) error { + return errors.New( + ErrMissingRegistrantCode, + errors.Alert, + []string{fmt.Sprintf("Model with name: %s does not have registrant information", modelName)}, + []string{"Meshery models are always registered in context of a registrant."}, + // there is only one cause for this error + []string{""}, + []string{"Make sure that the registrant information is present in the model definition"}, + ) +} + +func ErrRegisterEntity(err error, name, entity string) error { + return errors.New( + ErrRegisterEntityCode, + errors.Alert, + []string{fmt.Sprintf("Failed to register an entity of name: %s and type: %s into Meshery's registry", name, entity)}, + []string{err.Error()}, + []string{fmt.Sprintf("%s definition might be violating the definition schema", entity), fmt.Sprintf("%s might be missing model details", entity)}, + []string{fmt.Sprintf("ensure the %s definition follows the correct schema", entity), fmt.Sprintf("ensure %s definition belongs to correct model", entity)}, + ) +} + +func ErrGetEntity(err error) error { + return errors.New( + ErrGetEntityCode, + errors.Alert, + []string{"Could not parse the given data into any Meshery entity"}, + []string{err.Error()}, + []string{"Entity definition might not be in accordance with it's corresponding schema", "Might be an invalid/unsupported schemaVersion"}, + []string{"Ensure that the definition given is in accordance with it's schema", "Ensure that the schemaVersion used is valid and is supported by Meshery"}, + ) +} + +func ErrDirPkgUnitParseFail(dirpath string, err error) error { + return errors.New( + ErrDirPkgUnitParseFailCode, + errors.Alert, + []string{fmt.Sprintf("Directory at path: %s cannot be registered into Meshery", dirpath)}, + []string{fmt.Sprint(err.Error())}, + []string{"The directory might not have a valid model definition", "Might be some internal issues while walking the file tree"}, + []string{"Make sure that there is a valid model definition present in the directory. Meshery's registration pipeline currently does not support nested models, therefore the behaviour might be unexpected if it contains nested models.", "If there is an internal error, please try again after some time"}, + ) +} + +func ErrImportFailure(hostname string, failedMsg string) error { + return errors.New( + ErrImportFailureCode, + errors.Alert, + []string{fmt.Sprintf("Errors while registering entities for registrant: %s", hostname)}, + []string{failedMsg}, + []string{"Entity definition might not be in accordance with schema", "Entity version might not be supported by Meshery"}, + []string{"See the registration logs (found at $HOME/.meshery/logs/registry/registry-logs.log) to find out which Entity failed to be imported with more specific error information."}, + ) +} diff --git a/models/registration/interface.go b/models/registration/interface.go index c13f1394..a8721a38 100644 --- a/models/registration/interface.go +++ b/models/registration/interface.go @@ -4,21 +4,19 @@ import ( "github.com/layer5io/meshkit/models/meshmodel/entity" ) -/* - RegistrationErrorStore stores all the errors that does not break the registration process, but have to be reported nevertheless. +/* + RegistrationErrorStore stores all the errors that does not break the registration process, but have to be reported nevertheless. */ type RegistrationErrorStore interface { - AddInvalidDefinition(string,error) - InsertEntityRegError(hostname string, modelName string, entityType entity.EntityType, entityName string, err error) + AddInvalidDefinition(string, error) + InsertEntityRegError(hostname string, modelName string, entityType entity.EntityType, entityName string, err error) } // Anything that can be parsed into a packagingUnit is a RegisterableEntity in Meshery server type RegisterableEntity interface { - /* - 1. `err` - this is a breaking error, which signifies that the given entity is invalid and cannot be registered + /* + 1. `err` - this is a breaking error, which signifies that the given entity is invalid and cannot be registered 2. Errors encountered while parsing items into meshmodel entites are stored in the RegistrationErrorStore */ PkgUnit(RegistrationErrorStore) (packagingUnit, error) } - - diff --git a/models/registration/oci.go b/models/registration/oci.go index 4aa42f50..8c8af21b 100644 --- a/models/registration/oci.go +++ b/models/registration/oci.go @@ -1,16 +1,14 @@ package registration - -import( +import ( gcrv1 "github.com/google/go-containerregistry/pkg/v1" - ) type OCIImage struct { - _ gcrv1.Image + _ gcrv1.Image } -func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error){ +func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error) { pkg := packagingUnit{} return pkg, nil } diff --git a/models/registration/tar.go b/models/registration/tar.go index 66decb59..139a28bb 100644 --- a/models/registration/tar.go +++ b/models/registration/tar.go @@ -1,10 +1,10 @@ package registration type Tar struct { - _ string + _ string } -func (t Tar) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error){ +func (t Tar) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error) { pkg := packagingUnit{} return pkg, nil } From ac11bbde337776622e3253c617c39ae4dfbc32f9 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 13 Aug 2024 22:44:20 +0530 Subject: [PATCH 032/112] fix writeToSVGPath logic Signed-off-by: MUzairS15 --- models/registration/register.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/models/registration/register.go b/models/registration/register.go index b95e815e..d7b321b1 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -55,19 +55,26 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { rh.regErrStore.InsertEntityRegError(model.Registrant.Kind, "", entity.Model, model.Name, err) return } + if model.Metadata != nil { svgComplete := "" if model.Metadata.SvgComplete != nil { svgComplete = *model.Metadata.SvgComplete } + var svgCompletePath string + //Write SVG for models - WriteAndReplaceSVGWithFileSystemPath(model.Metadata.SvgColor, + model.Metadata.SvgColor, model.Metadata.SvgWhite, svgCompletePath = WriteAndReplaceSVGWithFileSystemPath(model.Metadata.SvgColor, model.Metadata.SvgWhite, svgComplete, rh.svgBaseDir, model.Name, model.Name, ) + if svgCompletePath != "" { + model.Metadata.SvgComplete = &svgCompletePath + } + } _, _, err := rh.regManager.RegisterEntity( connection.Connection{Kind: model.Registrant.Kind}, @@ -86,16 +93,19 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { // 2. Register components for _, comp := range pkg.components { comp.Model = model + if comp.Styles != nil { //Write SVG for components - WriteAndReplaceSVGWithFileSystemPath( + comp.Styles.SvgColor, comp.Styles.SvgWhite, comp.Styles.SvgComplete = WriteAndReplaceSVGWithFileSystemPath( comp.Styles.SvgColor, comp.Styles.SvgWhite, comp.Styles.SvgComplete, rh.svgBaseDir, comp.Model.Name, - comp.Component.Kind) //Write SVG on components + comp.Component.Kind, + ) } + _, _, err := rh.regManager.RegisterEntity( connection.Connection{Kind: model.Registrant.Kind}, &comp, From 74551a4bd83c08b58e0a903bb8237c0bd6a8820a Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 13 Aug 2024 23:02:45 +0530 Subject: [PATCH 033/112] encoding package Signed-off-by: MUzairS15 --- encoding/encoding.go | 51 ++++++++++++++++++++++++++++++++++++ encoding/error.go | 47 +++++++++++++++++++++++++++++++++ models/registration/utils.go | 22 ++++------------ utils/utils.go | 4 +-- 4 files changed, 105 insertions(+), 19 deletions(-) create mode 100644 encoding/encoding.go create mode 100644 encoding/error.go diff --git a/encoding/encoding.go b/encoding/encoding.go new file mode 100644 index 00000000..24fa378c --- /dev/null +++ b/encoding/encoding.go @@ -0,0 +1,51 @@ +package encoding + +import ( + "encoding/json" + + "gopkg.in/yaml.v2" +) + +// Unmarshal parses the JSON/YAML data and stores the result in the value pointed to by out +func Unmarshal(data []byte, out interface{}) error { + err := unmarshalJSON(data, out) + if err != nil { + err = unmarshalYAML(data, out) + if err != nil { + return err + } + } + return nil +} + +func unmarshalYAML(data []byte, result interface{}) error { + err := yaml.Unmarshal(data, result) + if err != nil { + return ErrDecodeYaml(err) + } + return nil +} + +func unmarshalJSON(data []byte, result interface{}) error { + + err := json.Unmarshal(data, result) + if err != nil { + if e, ok := err.(*json.SyntaxError); ok { + return ErrUnmarshalSyntax(err, e.Offset) + } + if e, ok := err.(*json.UnmarshalTypeError); ok { + return ErrUnmarshalType(err, e.Value) + } + if e, ok := err.(*json.UnsupportedTypeError); ok { + return ErrUnmarshalUnsupportedType(err, e.Type) + } + if e, ok := err.(*json.UnsupportedValueError); ok { + return ErrUnmarshalUnsupportedValue(err, e.Value) + } + if e, ok := err.(*json.InvalidUnmarshalError); ok { + return ErrUnmarshalInvalid(err, e.Type) + } + return ErrUnmarshal(err) + } + return nil +} diff --git a/encoding/error.go b/encoding/error.go new file mode 100644 index 00000000..7d100853 --- /dev/null +++ b/encoding/error.go @@ -0,0 +1,47 @@ +package encoding + +import ( + "reflect" + "strconv" + + "github.com/layer5io/meshkit/errors" +) + +const ( + ErrDecodeYamlCode = "" + ErrUnmarshalCode = "" + ErrUnmarshalInvalidCode = "" + ErrUnmarshalSyntaxCode = "" + ErrUnmarshalTypeCode = "" + ErrUnmarshalUnsupportedTypeCode = "" + ErrUnmarshalUnsupportedValueCode = "" +) + +// ErrDecodeYaml is the error when the yaml unmarshal fails +func ErrDecodeYaml(err error) error { + return errors.New(ErrDecodeYamlCode, errors.Alert, []string{"Error occurred while decoding YAML"}, []string{err.Error()}, []string{"Invalid object format"}, []string{"Make sure to input a valid YAML object"}) +} + +func ErrUnmarshal(err error) error { + return errors.New(ErrUnmarshalCode, errors.Alert, []string{"Unmarshal unknown error: "}, []string{err.Error()}, []string{"Invalid object format"}, []string{"Make sure to input a valid JSON object"}) +} + +func ErrUnmarshalInvalid(err error, typ reflect.Type) error { + return errors.New(ErrUnmarshalInvalidCode, errors.Alert, []string{"Unmarshal invalid error for type: ", typ.String()}, []string{err.Error()}, []string{"Invalid object format"}, []string{"Make sure to input a valid JSON object"}) +} + +func ErrUnmarshalSyntax(err error, offset int64) error { + return errors.New(ErrUnmarshalSyntaxCode, errors.Alert, []string{"Unmarshal syntax error at offest: ", strconv.Itoa(int(offset))}, []string{err.Error()}, []string{"Invalid object format"}, []string{"Make sure to input a valid JSON object"}) +} + +func ErrUnmarshalType(err error, value string) error { + return errors.New(ErrUnmarshalTypeCode, errors.Alert, []string{"Unmarshal type error at key: %s. Error: %s", value}, []string{err.Error()}, []string{"Invalid object format"}, []string{"Make sure to input a valid JSON object"}) +} + +func ErrUnmarshalUnsupportedType(err error, typ reflect.Type) error { + return errors.New(ErrUnmarshalUnsupportedTypeCode, errors.Alert, []string{"Unmarshal unsupported type error at key: ", typ.String()}, []string{err.Error()}, []string{"Invalid object format"}, []string{"Make sure to input a valid JSON object"}) +} + +func ErrUnmarshalUnsupportedValue(err error, value reflect.Value) error { + return errors.New(ErrUnmarshalUnsupportedValueCode, errors.Alert, []string{"Unmarshal unsupported value error at key: ", value.String()}, []string{err.Error()}, []string{"Invalid object format"}, []string{"Make sure to input a valid JSON object"}) +} diff --git a/models/registration/utils.go b/models/registration/utils.go index d3699fac..de8e1328 100644 --- a/models/registration/utils.go +++ b/models/registration/utils.go @@ -1,57 +1,45 @@ package registration import ( - "encoding/json" "fmt" + "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/meshery/schemas/models/v1alpha3" "github.com/meshery/schemas/models/v1alpha3/relationship" "github.com/meshery/schemas/models/v1beta1" "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/model" - "gopkg.in/yaml.v2" ) -func unmarshal(byt []byte, out interface{}) error { - err := json.Unmarshal(byt, out) - if err != nil { - err = yaml.Unmarshal(byt, out) - if err != nil { - return fmt.Errorf("Not a valid YAML or JSON") - } - } - return nil -} - // TODO: refactor this and use CUE func getEntity(byt []byte) (et entity.Entity, _ error) { type schemaVersion struct { SchemaVersion string `json:"schemaVersion" yaml:"schemaVersion"` } var sv schemaVersion - err := unmarshal(byt, &sv) + err := encoding.Unmarshal(byt, &sv) if err != nil || sv.SchemaVersion == "" { return nil, ErrGetEntity(fmt.Errorf("Does not contain versionmeta")) } switch sv.SchemaVersion { case v1beta1.ComponentSchemaVersion: var compDef component.ComponentDefinition - err := unmarshal(byt, &compDef) + err := encoding.Unmarshal(byt, &compDef) if err != nil { return nil, ErrGetEntity(fmt.Errorf("Invalid component definition: %s", err.Error())) } et = &compDef case v1beta1.ModelSchemaVersion: var model model.ModelDefinition - err := unmarshal(byt, &model) + err := encoding.Unmarshal(byt, &model) if err != nil { return nil, ErrGetEntity(fmt.Errorf("Invalid model definition: %s", err.Error())) } et = &model case v1alpha3.RelationshipSchemaVersion: var rel relationship.RelationshipDefinition - err := unmarshal(byt, &rel) + err := encoding.Unmarshal(byt, &rel) if err != nil { return nil, ErrGetEntity(fmt.Errorf("Invalid relationship definition: %s", err.Error())) } diff --git a/utils/utils.go b/utils/utils.go index 64d17c4e..73f1141f 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -38,7 +38,8 @@ func TransformMapKeys(input map[string]interface{}, transformFunc func(string) s return output } -// unmarshal returns parses the JSON config data and stores the value in the reference to result +// Deprecated: Use Unmarshal from encoding package. +// TODO: Replace the usages from all projects. func Unmarshal(obj string, result interface{}) error { obj = strings.TrimSpace(obj) err := json.Unmarshal([]byte(obj), result) @@ -408,7 +409,6 @@ func ExtractDomainFromURL(location string) string { } func IsInterfaceNil(val interface{}) bool { - fmt.Println("TEST 441 : ", val == nil) if val == nil { return true } From 196f34902abacc020153e8d8e1306c50a8ac2bf5 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 13 Aug 2024 23:07:44 +0530 Subject: [PATCH 034/112] update annotation prefix Signed-off-by: MUzairS15 --- models/meshmodel/core/v1beta1/host.go | 2 +- models/registration/dir.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/models/meshmodel/core/v1beta1/host.go b/models/meshmodel/core/v1beta1/host.go index 0ac38d14..0d2cdee2 100644 --- a/models/meshmodel/core/v1beta1/host.go +++ b/models/meshmodel/core/v1beta1/host.go @@ -59,7 +59,7 @@ func NewDependencyHandler(connectionKind string) (DependencyHandler, error) { type ArtifactHub struct{} -const MesheryAnnotationPrefix = "design.meshmodel.io" +const MesheryAnnotationPrefix = "design.meshery.io" func (ah ArtifactHub) HandleDependents(comp component.ComponentDefinition, kc *kubernetes.Client, isDeploy, performUpgrade bool) (summary string, err error) { sourceURI, err := utils.Cast[string](comp.Metadata.AdditionalProperties["source_uri"]) // should be part of registrant data(?) diff --git a/models/registration/dir.go b/models/registration/dir.go index dc5167c0..6a38725f 100644 --- a/models/registration/dir.go +++ b/models/registration/dir.go @@ -50,10 +50,12 @@ func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err e var e entity.Entity e, err = getEntity(byt) + fmt.Println("\n[GET ENTITY : ", err, path, "]") if err != nil { regErrStore.AddInvalidDefinition(path, err) return nil } + // set it to pkgunit switch e.Type() { case entity.Model: @@ -62,6 +64,7 @@ func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err e return nil } model, err := utils.Cast[*model.ModelDefinition](e) + // fmt.Println("INSIDE model 67 : ", model, err) if err != nil { regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) } @@ -71,12 +74,14 @@ func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err e if err != nil { regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) } + // fmt.Println("INSIDE comp 77 : ", comp.Component.Kind, err) pkg.components = append(pkg.components, *comp) case entity.RelationshipDefinition: rel, err := utils.Cast[*relationship.RelationshipDefinition](e) if err != nil { regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) } + // fmt.Println("INSIDE rel 84 : ", rel.Kind, rel.RelationshipType, rel.SubType, err) pkg.relationships = append(pkg.relationships, *rel) } return nil From c5fa81b669d3f9d22c80f4d48629769c61727a3b Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 14 Aug 2024 12:12:19 +0530 Subject: [PATCH 035/112] fix cache hit logic for svg writer Signed-off-by: MUzairS15 --- models/registration/svg_helper.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/registration/svg_helper.go b/models/registration/svg_helper.go index 086e5673..5122c795 100644 --- a/models/registration/svg_helper.go +++ b/models/registration/svg_helper.go @@ -55,7 +55,7 @@ func WriteAndReplaceSVGWithFileSystemPath(svgColor, svgWhite, svgComplete string return } svgColorPath = getRelativePathForAPI(baseDir, filepath.Join(dirname, "color", filename+"-color.svg")) //Replace the actual SVG with path to SVG - writeHashCheckSVG(hashString, svgColor) + writeHashCheckSVG(hashString, svgColorPath) } White: @@ -86,7 +86,7 @@ White: return } svgWhitePath = getRelativePathForAPI(baseDir, filepath.Join(dirname, "white", filename+"-white.svg")) //Replace the actual SVG with path to SVG - writeHashCheckSVG(hashString, svgWhite) + writeHashCheckSVG(hashString, svgWhitePath) } Complete: @@ -117,7 +117,7 @@ Complete: return } svgCompletePath = getRelativePathForAPI(baseDir, filepath.Join(dirname, "complete", filename+"-complete.svg")) //Replace the actual SVG with path to SVG - writeHashCheckSVG(hashString, svgComplete) + writeHashCheckSVG(hashString, svgCompletePath) } return From 6ab7fca226615ae2fbbc701c6dfa999d3eaaa889 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 14 Aug 2024 12:12:42 +0530 Subject: [PATCH 036/112] use complete registrant information from model definition Signed-off-by: MUzairS15 --- models/registration/register.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/models/registration/register.go b/models/registration/register.go index d7b321b1..dd208cdb 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -76,8 +76,10 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { } } + + model.Registrant.Status = connection.Registered _, _, err := rh.regManager.RegisterEntity( - connection.Connection{Kind: model.Registrant.Kind}, + model.Registrant, &model, ) @@ -107,7 +109,7 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { } _, _, err := rh.regManager.RegisterEntity( - connection.Connection{Kind: model.Registrant.Kind}, + model.Registrant, &comp, ) if err != nil { @@ -119,7 +121,7 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { // 3. Register relationships for _, rel := range pkg.relationships { rel.Model = model - _, _, err := rh.regManager.RegisterEntity(connection.Connection{Kind: model.Registrant.Kind}, &rel) + _, _, err := rh.regManager.RegisterEntity(model.Registrant, &rel) if err != nil { err = ErrRegisterEntity(err, string(rel.Type()), string(rel.Kind)) rh.regErrStore.InsertEntityRegError(hostname, modelName, entity.RelationshipDefinition, rel.Id.String(), err) From 135b189d452378e0c57e5d29946f5403069b00b8 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 14 Aug 2024 12:13:01 +0530 Subject: [PATCH 037/112] fix regsitrants API Signed-off-by: MUzairS15 --- models/meshmodel/registry/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index 91606a61..459ab7c3 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -149,8 +149,8 @@ func (rm *RegistryManager) GetRegistrants(f *models.HostFilter) ([]models.MeshMo "COUNT(CASE WHEN r.type = 'model' THEN 1 END) AS models," + "COUNT(CASE WHEN r.type = 'relationship' THEN 1 END) AS relationships, " + "COUNT(CASE WHEN r.type = 'policy' THEN 1 END) AS policies"). - Joins("LEFT JOIN registries r ON c.id = r.registrant_id") - // Group("c.id, c.hostname, h.port") // required + Joins("LEFT JOIN registries r ON c.id = r.registrant_id"). + Group("c.id") if f.DisplayName != "" { query = query.Where("kind LIKE ?", "%"+f.DisplayName+"%") From a4ceb8e033f055a2590b2fc3bf676d00b53f3205 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 14 Aug 2024 12:13:10 +0530 Subject: [PATCH 038/112] remove debug logs Signed-off-by: MUzairS15 --- models/registration/dir.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/models/registration/dir.go b/models/registration/dir.go index 6a38725f..fb7db466 100644 --- a/models/registration/dir.go +++ b/models/registration/dir.go @@ -50,7 +50,6 @@ func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err e var e entity.Entity e, err = getEntity(byt) - fmt.Println("\n[GET ENTITY : ", err, path, "]") if err != nil { regErrStore.AddInvalidDefinition(path, err) return nil @@ -64,7 +63,6 @@ func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err e return nil } model, err := utils.Cast[*model.ModelDefinition](e) - // fmt.Println("INSIDE model 67 : ", model, err) if err != nil { regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) } @@ -74,14 +72,12 @@ func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err e if err != nil { regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) } - // fmt.Println("INSIDE comp 77 : ", comp.Component.Kind, err) pkg.components = append(pkg.components, *comp) case entity.RelationshipDefinition: rel, err := utils.Cast[*relationship.RelationshipDefinition](e) if err != nil { regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) } - // fmt.Println("INSIDE rel 84 : ", rel.Kind, rel.RelationshipType, rel.SubType, err) pkg.relationships = append(pkg.relationships, *rel) } return nil From a70206911067a6b70d93ad05761485ae8979dfe7 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 14 Aug 2024 12:13:22 +0530 Subject: [PATCH 039/112] update response struct Signed-off-by: MUzairS15 --- .../core/policies/rego_policy_relationship.go | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index 4f26ddca..06cc5234 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -49,9 +49,10 @@ func NewRegoInstance(policyDir string, regManager *registry.RegistryManager) (*R } // RegoPolicyHandler takes the required inputs and run the query against all the policy files provided -func (r *Rego) RegoPolicyHandler(designFile pattern.PatternFile, regoQueryString string, relationshipsToEvalaute ...string) (interface{}, error) { +func (r *Rego) RegoPolicyHandler(designFile pattern.PatternFile, regoQueryString string, relationshipsToEvalaute ...string) (pattern.EvaluationResponse, error) { + var evaluationResponse pattern.EvaluationResponse if r == nil { - return nil, ErrEval(fmt.Errorf("policy engine is not yet ready")) + return evaluationResponse, ErrEval(fmt.Errorf("policy engine is not yet ready")) } regoEngine, err := rego.New( rego.Query(regoQueryString), @@ -61,37 +62,37 @@ func (r *Rego) RegoPolicyHandler(designFile pattern.PatternFile, regoQueryString ).PrepareForEval(r.ctx) if err != nil { logrus.Error("error preparing for evaluation", err) - return nil, ErrPrepareForEval(err) + return evaluationResponse, ErrPrepareForEval(err) } eval_result, err := regoEngine.Eval(r.ctx, rego.EvalInput(designFile)) if err != nil { - return nil, ErrEval(err) + return evaluationResponse, ErrEval(err) } if len(eval_result) == 0 { - return nil, ErrEval(fmt.Errorf("evaluation results are empty")) + return evaluationResponse, ErrEval(fmt.Errorf("evaluation results are empty")) } if len(eval_result[0].Expressions) == 0 { - return nil, ErrEval(fmt.Errorf("evaluation results are empty")) + return evaluationResponse, ErrEval(fmt.Errorf("evaluation results are empty")) } result, err := utils.Cast[map[string]interface{}](eval_result[0].Expressions[0].Value) if err != nil { - return nil, ErrEval(err) + return evaluationResponse, ErrEval(err) } evalResults, ok := result["evaluate"] if !ok { - return nil, ErrEval(fmt.Errorf("evaluation results are empty")) + return evaluationResponse, ErrEval(fmt.Errorf("evaluation results are empty")) } evaluatedPatternFile, err := utils.MarshalAndUnmarshal[interface{}, pattern.PatternFile](evalResults) if err != nil { - return nil, err + return evaluationResponse, err } - - return evaluatedPatternFile, nil + evaluationResponse.Design = evaluatedPatternFile + return evaluationResponse, nil } From 1243ede2fbeedb054bc5f323f7e513f03bc524c7 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Wed, 14 Aug 2024 18:43:51 +0530 Subject: [PATCH 040/112] add check for missing metadata w/signoff Signed-off-by: Jougan-0 --- generators/artifacthub/package.go | 4 ++++ generators/github/package.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index 3c018813..61b3d8be 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -12,6 +12,7 @@ import ( "github.com/layer5io/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1/category" _component "github.com/meshery/schemas/models/v1beta1/component" + "github.com/meshery/schemas/models/v1beta1/model" "gopkg.in/yaml.v2" ) @@ -53,6 +54,9 @@ func (pkg AhPackage) GenerateComponents() ([]_component.ComponentDefinition, err if err != nil { continue } + if comp.Model.Metadata == nil { + comp.Model.Metadata = &model.ModelDefinition_Metadata{} + } if comp.Model.Metadata.AdditionalProperties == nil { comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) diff --git a/generators/github/package.go b/generators/github/package.go index ef09aae8..0c632ef9 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -9,6 +9,7 @@ import ( "github.com/layer5io/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1/category" _component "github.com/meshery/schemas/models/v1beta1/component" + "github.com/meshery/schemas/models/v1beta1/model" ) type GitHubPackage struct { @@ -40,6 +41,9 @@ func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, if err != nil { continue } + if comp.Model.Metadata == nil { + comp.Model.Metadata = &model.ModelDefinition_Metadata{} + } if comp.Model.Metadata.AdditionalProperties == nil { comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) } From bc379ffa6ff0bd136b8ab3dd9f4f509a9e50a8a6 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 14 Aug 2024 19:32:02 +0530 Subject: [PATCH 041/112] use mehskit's encoding package Signed-off-by: MUzairS15 --- models/patterns/pattern.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/patterns/pattern.go b/models/patterns/pattern.go index 5d76bb8b..5b933132 100644 --- a/models/patterns/pattern.go +++ b/models/patterns/pattern.go @@ -2,9 +2,9 @@ package patterns import ( "github.com/Masterminds/semver/v3" + "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/utils" "github.com/meshery/schemas/models/v1beta1/pattern" - "gopkg.in/yaml.v2" ) func GetNextVersion(p *pattern.PatternFile) (string, error) { @@ -29,7 +29,7 @@ func AssignVersion(p *pattern.PatternFile) { } func GetPatternFormat(patternFile string) (*pattern.PatternFile, error) { pattern := pattern.PatternFile{} - err := yaml.Unmarshal([]byte(patternFile), &pattern) + err := encoding.Unmarshal([]byte(patternFile), &pattern) if err != nil { err = utils.ErrDecodeYaml(err) return nil, err From d5bd69177c1e2688e886ec9cd65877cd02f9afba Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Fri, 16 Aug 2024 11:20:34 +0530 Subject: [PATCH 042/112] update schemas package Signed-off-by: MUzairS15 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6c8609d1..515596ec 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/google/uuid v1.6.0 github.com/kubernetes/kompose v1.31.1 github.com/layer5io/meshery-operator v0.7.0 - github.com/meshery/schemas v0.7.17 + github.com/meshery/schemas v0.7.18 github.com/nats-io/nats.go v1.31.0 github.com/open-policy-agent/opa v0.67.1 github.com/opencontainers/image-spec v1.1.0 diff --git a/go.sum b/go.sum index b6991647..f8aac588 100644 --- a/go.sum +++ b/go.sum @@ -608,8 +608,8 @@ github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/meshery/kompose v1.0.1 h1:lg8B/pkLh6762jeFsQATD8UJZZwXZf/aviC3/dzw78A= github.com/meshery/kompose v1.0.1/go.mod h1:TWhWTEMbJBUzENf4JTEtBmZRFm/r8n0nS6v4/nSD2vA= -github.com/meshery/schemas v0.7.17 h1:NOfPAECsaU0l5hASDMlu6yPBlwLkGn9CKUq9umjkCE0= -github.com/meshery/schemas v0.7.17/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= +github.com/meshery/schemas v0.7.18 h1:gXHDiPmlLKywfaE3nm5YRdjhWRMTUEsfVvZlJjB93Ug= +github.com/meshery/schemas v0.7.18/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= From 85c93d68a85fba78325f0317b310e3f1d983111f Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Fri, 16 Aug 2024 16:54:45 +0530 Subject: [PATCH 043/112] add marshal functions Signed-off-by: MUzairS15 --- encoding/encoding.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/encoding/encoding.go b/encoding/encoding.go index 24fa378c..8f980de5 100644 --- a/encoding/encoding.go +++ b/encoding/encoding.go @@ -3,6 +3,7 @@ package encoding import ( "encoding/json" + "github.com/layer5io/meshkit/utils" "gopkg.in/yaml.v2" ) @@ -49,3 +50,15 @@ func unmarshalJSON(data []byte, result interface{}) error { } return nil } + +func Marshal(in interface{}) ([]byte, error) { + result, err := json.Marshal(in) + if err != nil { + result, err = yaml.Marshal(in) + if err != nil { + return nil, utils.ErrMarshal(err) + } + } + + return result, nil +} From d648931e9ac59603d3f4fc4ef204522d65378a79 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Fri, 16 Aug 2024 16:54:59 +0530 Subject: [PATCH 044/112] remove debug logs Signed-off-by: MUzairS15 --- models/meshmodel/registry/registry.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index 459ab7c3..cc9d1c1b 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -81,13 +81,11 @@ func (rm *RegistryManager) Cleanup() { func (rm *RegistryManager) RegisterEntity(h connection.Connection, en entity.Entity) (bool, bool, error) { registrantID, err := h.Create(rm.db) if err != nil { - fmt.Println("REGISTRANT ERROR ") return true, false, err } entityID, err := en.Create(rm.db, registrantID) if err != nil { - fmt.Println("COMP ERROR ") return false, true, err } id, _ := uuid.NewV4() From bee904dca9151d02479388b10af003bddcc3f409 Mon Sep 17 00:00:00 2001 From: Alero Awani Date: Sat, 17 Aug 2024 15:35:37 +0100 Subject: [PATCH 045/112] add default version Signed-off-by: Alero Awani --- utils/helm/helm.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/utils/helm/helm.go b/utils/helm/helm.go index 7c9b69f9..18ca24ed 100644 --- a/utils/helm/helm.go +++ b/utils/helm/helm.go @@ -17,7 +17,7 @@ import ( ) func extractSemVer(versionConstraint string) string { - reg := regexp.MustCompile(`v([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$`) + reg := regexp.MustCompile(`v?([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$`) match := reg.Find([]byte(versionConstraint)) if match != nil { return string(match) @@ -34,14 +34,20 @@ func DryRunHelmChart(chart *chart.Chart) ([]byte, error) { act.DryRun = true act.IncludeCRDs = true act.ClientOnly = true + + version := "v1.29.5" // Default version + if chart.Metadata.KubeVersion != "" { - version := extractSemVer(chart.Metadata.KubeVersion) - if version != "" { - act.KubeVersion = &chartutil.KubeVersion{ - Version: version, - } - } + extractedVersion := extractSemVer(chart.Metadata.KubeVersion) + if extractedVersion != "" { + version = extractedVersion + } + } + + act.KubeVersion = &chartutil.KubeVersion{ + Version: version, } + rel, err := act.Run(chart, nil) if err != nil { return nil, ErrDryRunHelmChart(err, chart.Name()) From 03fd3d2c8e81525b0b40bfd68e1428fd08e807ec Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Sun, 18 Aug 2024 23:05:12 +0530 Subject: [PATCH 046/112] directly send the response instead of processing Signed-off-by: MUzairS15 --- models/meshmodel/core/policies/rego_policy_relationship.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index 06cc5234..8b2c60e0 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -88,11 +88,11 @@ func (r *Rego) RegoPolicyHandler(designFile pattern.PatternFile, regoQueryString return evaluationResponse, ErrEval(fmt.Errorf("evaluation results are empty")) } - evaluatedPatternFile, err := utils.MarshalAndUnmarshal[interface{}, pattern.PatternFile](evalResults) + evaluationResponse, err = utils.MarshalAndUnmarshal[interface{}, pattern.EvaluationResponse](evalResults) if err != nil { return evaluationResponse, err } - evaluationResponse.Design = evaluatedPatternFile + return evaluationResponse, nil } From 29558eee45329f40e270e8d9d9939d98fd0702bb Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 19 Aug 2024 12:39:10 +0530 Subject: [PATCH 047/112] update schemas to v0.7.19 Signed-off-by: MUzairS15 --- go.mod | 4 +++- go.sum | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 515596ec..b914a6d2 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/google/uuid v1.6.0 github.com/kubernetes/kompose v1.31.1 github.com/layer5io/meshery-operator v0.7.0 - github.com/meshery/schemas v0.7.18 + github.com/meshery/schemas v0.7.19 github.com/nats-io/nats.go v1.31.0 github.com/open-policy-agent/opa v0.67.1 github.com/opencontainers/image-spec v1.1.0 @@ -260,3 +260,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) + +replace github.com/meshery/schemas => ../schemas diff --git a/go.sum b/go.sum index f8aac588..58dff424 100644 --- a/go.sum +++ b/go.sum @@ -608,8 +608,6 @@ github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/meshery/kompose v1.0.1 h1:lg8B/pkLh6762jeFsQATD8UJZZwXZf/aviC3/dzw78A= github.com/meshery/kompose v1.0.1/go.mod h1:TWhWTEMbJBUzENf4JTEtBmZRFm/r8n0nS6v4/nSD2vA= -github.com/meshery/schemas v0.7.18 h1:gXHDiPmlLKywfaE3nm5YRdjhWRMTUEsfVvZlJjB93Ug= -github.com/meshery/schemas v0.7.18/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= From ea8bd93af5806da6479a1764b927321876a03338 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 19 Aug 2024 12:41:17 +0530 Subject: [PATCH 048/112] run gofmt Signed-off-by: MUzairS15 --- models/registration/interface.go | 2 +- models/registration/register.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/registration/interface.go b/models/registration/interface.go index a8721a38..3f9f580b 100644 --- a/models/registration/interface.go +++ b/models/registration/interface.go @@ -5,7 +5,7 @@ import ( ) /* - RegistrationErrorStore stores all the errors that does not break the registration process, but have to be reported nevertheless. +RegistrationErrorStore stores all the errors that does not break the registration process, but have to be reported nevertheless. */ type RegistrationErrorStore interface { AddInvalidDefinition(string, error) diff --git a/models/registration/register.go b/models/registration/register.go index dd208cdb..50f86639 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -76,7 +76,7 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { } } - + model.Registrant.Status = connection.Registered _, _, err := rh.regManager.RegisterEntity( model.Registrant, From d30a4c76151159d0c6618b5df3553f6abb5c0524 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Mon, 19 Aug 2024 13:45:52 +0530 Subject: [PATCH 049/112] fix: fetch styles from db w/signoff Signed-off-by: Jougan-0 --- models/meshmodel/registry/registry.go | 4 -- .../registry/v1beta1/model_filter.go | 42 ++----------------- 2 files changed, 4 insertions(+), 42 deletions(-) diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index cc9d1c1b..e934df61 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -197,10 +197,6 @@ func (rm *RegistryManager) GetEntities(f entity.Filter) ([]entity.Entity, int64, return f.Get(rm.db) } -func (rm *RegistryManager) GetEntityById(f entity.Filter) (entity.Entity, error) { - return f.GetById(rm.db) -} - func HostnameToPascalCase(input string) string { parts := strings.Split(input, ".") caser := cases.Title(language.English) diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index 1819f1d2..9a806dc3 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -3,7 +3,6 @@ package v1beta1 import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/meshery/schemas/models/v1alpha3/relationship" "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/model" @@ -50,42 +49,6 @@ func countUniqueModels(models []model.ModelDefinition) int { return len(set) } -func (mf *ModelFilter) GetById(db *database.Handler) (entity.Entity, error) { - m := &model.ModelDefinition{} - - // Retrieve the model by ID - err := db.First(m, "id = ?", mf.Id).Error - if err != nil { - return nil, registry.ErrGetById(err, mf.Id) - } - - // Include components if requested - if mf.Components { - var components []component.ComponentDefinition - componentFinder := db.Model(&component.ComponentDefinition{}). - Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version"). - Where("component_definition_dbs.model_id = ?", m.Id) - if err := componentFinder.Scan(&components).Error; err != nil { - return nil, err - } - m.Components = components - } - - // Include relationships if requested - if mf.Relationships { - var relationships []relationship.RelationshipDefinition - relationshipFinder := db.Model(&relationship.RelationshipDefinition{}). - Select("relationship_definition_dbs.*"). - Where("relationship_definition_dbs.model_id = ?", m.Id) - if err := relationshipFinder.Scan(&relationships).Error; err != nil { - return nil, err - } - m.Relationships = relationships - } - - return m, nil -} - func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { var modelWithCategories []model.ModelDefinition @@ -102,6 +65,9 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e var includeComponents, includeRelationships bool if mf.Greedy { + if mf.Id != "" { + finder = finder.First("model_dbs.id = ?", mf.Id) + } if mf.Name != "" && mf.DisplayName != "" { finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%") } else if mf.Name != "" { @@ -180,7 +146,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e if includeComponents { var components []component.ComponentDefinition finder := db.Model(&component.ComponentDefinition{}). - Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version"). + Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version,component_definition_dbs.styles"). Where("component_definition_dbs.model_id = ?", _modelDB.Id) if err := finder.Scan(&components).Error; err != nil { return nil, 0, 0, err From 1483c4b306ee7462d3c5528bafe4e4cf4492e1e4 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Mon, 19 Aug 2024 13:47:57 +0530 Subject: [PATCH 050/112] fix liniting w/signoff Signed-off-by: Jougan-0 --- go.mod | 2 -- go.sum | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index b914a6d2..d778a3fe 100644 --- a/go.mod +++ b/go.mod @@ -260,5 +260,3 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) - -replace github.com/meshery/schemas => ../schemas diff --git a/go.sum b/go.sum index 58dff424..59233ba0 100644 --- a/go.sum +++ b/go.sum @@ -608,6 +608,8 @@ github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/meshery/kompose v1.0.1 h1:lg8B/pkLh6762jeFsQATD8UJZZwXZf/aviC3/dzw78A= github.com/meshery/kompose v1.0.1/go.mod h1:TWhWTEMbJBUzENf4JTEtBmZRFm/r8n0nS6v4/nSD2vA= +github.com/meshery/schemas v0.7.19 h1:HS+PCZ3ZXbs7cv0JSUNW3kYnbypt3WqZjQ7fMQPy8uo= +github.com/meshery/schemas v0.7.19/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= From 5d1230f96d588d92dcb403b224bca523ea974124 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Mon, 19 Aug 2024 17:32:37 +0530 Subject: [PATCH 051/112] missing function w/signoff Signed-off-by: Jougan-0 --- .../registry/v1beta1/model_filter.go | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index 9a806dc3..86971485 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -3,6 +3,7 @@ package v1beta1 import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/meshery/schemas/models/v1alpha3/relationship" "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/model" @@ -48,6 +49,41 @@ func countUniqueModels(models []model.ModelDefinition) int { } return len(set) } +func (mf *ModelFilter) GetById(db *database.Handler) (entity.Entity, error) { + m := &model.ModelDefinition{} + + // Retrieve the model by ID + err := db.First(m, "id = ?", mf.Id).Error + if err != nil { + return nil, registry.ErrGetById(err, mf.Id) + } + + // Include components if requested + if mf.Components { + var components []component.ComponentDefinition + componentFinder := db.Model(&component.ComponentDefinition{}). + Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version"). + Where("component_definition_dbs.model_id = ?", m.Id) + if err := componentFinder.Scan(&components).Error; err != nil { + return nil, err + } + m.Components = components + } + + // Include relationships if requested + if mf.Relationships { + var relationships []relationship.RelationshipDefinition + relationshipFinder := db.Model(&relationship.RelationshipDefinition{}). + Select("relationship_definition_dbs.*"). + Where("relationship_definition_dbs.model_id = ?", m.Id) + if err := relationshipFinder.Scan(&relationships).Error; err != nil { + return nil, err + } + m.Relationships = relationships + } + + return m, nil +} func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { From 96522532c5f76dde1e97b7f957c3591588df9bd5 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 20 Aug 2024 17:42:40 +0530 Subject: [PATCH 052/112] use source_uri from model metadata Signed-off-by: MUzairS15 --- models/meshmodel/core/v1beta1/host.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/models/meshmodel/core/v1beta1/host.go b/models/meshmodel/core/v1beta1/host.go index 0d2cdee2..fe7d1c61 100644 --- a/models/meshmodel/core/v1beta1/host.go +++ b/models/meshmodel/core/v1beta1/host.go @@ -62,8 +62,8 @@ type ArtifactHub struct{} const MesheryAnnotationPrefix = "design.meshery.io" func (ah ArtifactHub) HandleDependents(comp component.ComponentDefinition, kc *kubernetes.Client, isDeploy, performUpgrade bool) (summary string, err error) { - sourceURI, err := utils.Cast[string](comp.Metadata.AdditionalProperties["source_uri"]) // should be part of registrant data(?) - if err != nil { + sourceURI, ok := comp.Model.Metadata.AdditionalProperties["source_uri"] // should be part of registrant data(?) + if !ok { return summary, err } @@ -72,9 +72,14 @@ func (ah ArtifactHub) HandleDependents(comp component.ComponentDefinition, kc *k act = kubernetes.INSTALL } + _sourceURI, err := utils.Cast[string](sourceURI) + if err != nil { + return summary, err + } + if sourceURI != "" { err = kc.ApplyHelmChart(kubernetes.ApplyHelmChartConfig{ - URL: sourceURI, + URL: _sourceURI, Namespace: comp.Configuration["namespace"].(string), CreateNamespace: true, Action: act, From 2dea93949a08c60b8041a90b8f113b2fe7c0d136 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 20 Aug 2024 18:05:03 +0530 Subject: [PATCH 053/112] remove debug logs Signed-off-by: MUzairS15 --- models/meshmodel/registry/registry.go | 1 - models/meshmodel/registry/v1beta1/component_filter.go | 3 --- utils/kubernetes/apply-manifest.go | 2 -- 3 files changed, 6 deletions(-) diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index cc9d1c1b..ad1b03cc 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -177,7 +177,6 @@ func (rm *RegistryManager) GetRegistrants(f *models.HostFilter) ([]models.MeshMo var response []models.MeshModelHostsWithEntitySummary - fmt.Println("result--------------------------", result) for _, r := range result { res := models.MeshModelHostsWithEntitySummary{ Connection: r.Connection, diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index 3265abfc..76a6fb57 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -1,8 +1,6 @@ package v1beta1 import ( - "fmt" - "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/models/meshmodel/registry" @@ -139,7 +137,6 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti err := finder. Scan(&componentDefinitionsWithModel).Error if err != nil { - fmt.Println("line 123 : =+++++++++++++++", err) return nil, 0, 0, err } diff --git a/utils/kubernetes/apply-manifest.go b/utils/kubernetes/apply-manifest.go index 91a4e8f9..f1d1beac 100644 --- a/utils/kubernetes/apply-manifest.go +++ b/utils/kubernetes/apply-manifest.go @@ -2,7 +2,6 @@ package kubernetes import ( "context" - "fmt" "strings" v1 "k8s.io/api/core/v1" @@ -95,7 +94,6 @@ func (client *Client) ApplyManifest(contents []byte, recvOptions ApplyOptions) e return err } } else { - fmt.Println("Creating object", object) _, err = createObject(helper, options.Namespace, object, options.Update) if err != nil && !kubeerror.IsAlreadyExists(err) { if recvOptions.IgnoreErrors { From 2f78114f0bbc054c11c75ff65c6733bd0ea30f0b Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Tue, 20 Aug 2024 19:25:14 +0530 Subject: [PATCH 054/112] remove unwanted logs w/signoff Signed-off-by: Jougan-0 --- models/meshmodel/registry/registry.go | 1 - 1 file changed, 1 deletion(-) diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index cc9d1c1b..ad1b03cc 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -177,7 +177,6 @@ func (rm *RegistryManager) GetRegistrants(f *models.HostFilter) ([]models.MeshMo var response []models.MeshModelHostsWithEntitySummary - fmt.Println("result--------------------------", result) for _, r := range result { res := models.MeshModelHostsWithEntitySummary{ Connection: r.Connection, From 0bc3bc4fbb1a9981423253d55c3949c4650206d5 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 21 Aug 2024 15:38:49 +0530 Subject: [PATCH 055/112] upgrade operator installation if already exists Signed-off-by: MUzairS15 --- models/controllers/helpers.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/controllers/helpers.go b/models/controllers/helpers.go index a1454ce3..7add4f51 100644 --- a/models/controllers/helpers.go +++ b/models/controllers/helpers.go @@ -81,6 +81,8 @@ func applyOperatorHelmChart(chartRepo string, client mesherykube.Client, meshery Action: act, // Setting override values OverrideValues: overrides, + + UpgradeIfInstalled: true, }) if err != nil { return err From 4bf0a52687b75356ce2aa3b89b392fdbabc66ff9 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 21 Aug 2024 15:41:39 +0530 Subject: [PATCH 056/112] chore: upgrade meshery/schemas Signed-off-by: MUzairS15 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 515596ec..412607c0 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/google/uuid v1.6.0 github.com/kubernetes/kompose v1.31.1 github.com/layer5io/meshery-operator v0.7.0 - github.com/meshery/schemas v0.7.18 + github.com/meshery/schemas v0.7.20 github.com/nats-io/nats.go v1.31.0 github.com/open-policy-agent/opa v0.67.1 github.com/opencontainers/image-spec v1.1.0 diff --git a/go.sum b/go.sum index f8aac588..8c11bbdd 100644 --- a/go.sum +++ b/go.sum @@ -608,8 +608,8 @@ github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/meshery/kompose v1.0.1 h1:lg8B/pkLh6762jeFsQATD8UJZZwXZf/aviC3/dzw78A= github.com/meshery/kompose v1.0.1/go.mod h1:TWhWTEMbJBUzENf4JTEtBmZRFm/r8n0nS6v4/nSD2vA= -github.com/meshery/schemas v0.7.18 h1:gXHDiPmlLKywfaE3nm5YRdjhWRMTUEsfVvZlJjB93Ug= -github.com/meshery/schemas v0.7.18/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= +github.com/meshery/schemas v0.7.20 h1:1v5+PKjwq/jq0/dayVNJVDbsrh8GXI5QmDYvEBPDdac= +github.com/meshery/schemas v0.7.20/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= From e9e7c2f870924a3c52e85c2241c84b630459904e Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 21 Aug 2024 16:26:59 +0530 Subject: [PATCH 057/112] peform helm dry run based on the k8s model available in the registry Signed-off-by: MUzairS15 --- generators/github/git_repo.go | 2 +- generators/github/url.go | 2 +- utils/helm/helm.go | 28 ++++++++++++++++++---------- utils/kubernetes/apply-helm-chart.go | 4 ++++ utils/kubernetes/helm.go | 2 +- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/generators/github/git_repo.go b/generators/github/git_repo.go index 47b40529..e24181db 100644 --- a/generators/github/git_repo.go +++ b/generators/github/git_repo.go @@ -105,7 +105,7 @@ func fileInterceptor(br *bufio.Writer) walker.FileInterceptor { // Add more calrifying commment and entry inside docs. func dirInterceptor(br *bufio.Writer) walker.DirInterceptor { return func(d walker.Directory) error { - err := helm.ConvertToK8sManifest(d.Path, br) + err := helm.ConvertToK8sManifest(d.Path, "", br) if err != nil { return err } diff --git a/generators/github/url.go b/generators/github/url.go index 89541251..6ca4b94d 100644 --- a/generators/github/url.go +++ b/generators/github/url.go @@ -77,7 +77,7 @@ func ProcessContent(w io.Writer, downloadDirPath, downloadfilePath string) error } err = utils.ProcessContent(downloadDirPath, func(path string) error { - err = helm.ConvertToK8sManifest(path, w) + err = helm.ConvertToK8sManifest(path, "", w) if err != nil { return err } diff --git a/utils/helm/helm.go b/utils/helm/helm.go index 7c9b69f9..caf8463f 100644 --- a/utils/helm/helm.go +++ b/utils/helm/helm.go @@ -26,7 +26,7 @@ func extractSemVer(versionConstraint string) string { } // DryRun a given helm chart to convert into k8s manifest -func DryRunHelmChart(chart *chart.Chart) ([]byte, error) { +func DryRunHelmChart(chart *chart.Chart, kubernetesVersion string) ([]byte, error) { actconfig := new(action.Configuration) act := action.NewInstall(actconfig) act.ReleaseName = chart.Metadata.Name @@ -34,14 +34,22 @@ func DryRunHelmChart(chart *chart.Chart) ([]byte, error) { act.DryRun = true act.IncludeCRDs = true act.ClientOnly = true + + kubeVersion := kubernetesVersion if chart.Metadata.KubeVersion != "" { - version := extractSemVer(chart.Metadata.KubeVersion) - if version != "" { - act.KubeVersion = &chartutil.KubeVersion{ - Version: version, - } + extractedVersion := extractSemVer(chart.Metadata.KubeVersion) + + if extractedVersion != "" { + kubeVersion = extractedVersion } } + + if kubeVersion != "" { + act.KubeVersion = &chartutil.KubeVersion{ + Version: kubeVersion, + } + } + rel, err := act.Run(chart, nil) if err != nil { return nil, ErrDryRunHelmChart(err, chart.Name()) @@ -55,7 +63,7 @@ func DryRunHelmChart(chart *chart.Chart) ([]byte, error) { } // Takes in the directory and converts HelmCharts/multiple manifests into a single K8s manifest -func ConvertToK8sManifest(path string, w io.Writer) error { +func ConvertToK8sManifest(path, kubeVersion string, w io.Writer) error { info, err := os.Stat(path) if err != nil { return utils.ErrReadDir(err, path) @@ -65,7 +73,7 @@ func ConvertToK8sManifest(path string, w io.Writer) error { helmChartPath, _ = strings.CutSuffix(path, filepath.Base(path)) } if IsHelmChart(helmChartPath) { - err := LoadHelmChart(helmChartPath, w, true) + err := LoadHelmChart(helmChartPath, w, true, kubeVersion) if err != nil { return err } @@ -124,7 +132,7 @@ func IsHelmChart(dirPath string) bool { return true } -func LoadHelmChart(path string, w io.Writer, extractOnlyCrds bool) error { +func LoadHelmChart(path string, w io.Writer, extractOnlyCrds bool, kubeVersion string) error { var errs []error chart, err := loader.Load(path) if err != nil { @@ -145,7 +153,7 @@ func LoadHelmChart(path string, w io.Writer, extractOnlyCrds bool) error { _, _ = w.Write([]byte("\n---\n")) } } else { - manifests, err := DryRunHelmChart(chart) + manifests, err := DryRunHelmChart(chart, kubeVersion) if err != nil { return ErrLoadHelmChart(err, path) } diff --git a/utils/kubernetes/apply-helm-chart.go b/utils/kubernetes/apply-helm-chart.go index d77ad292..7dcc61e9 100644 --- a/utils/kubernetes/apply-helm-chart.go +++ b/utils/kubernetes/apply-helm-chart.go @@ -122,6 +122,9 @@ type ApplyHelmChartConfig struct { // SkipCRDs while installation SkipCRDs bool + // Kubernetes version against which the DryRun is performed + KubernetesVersion string + // Upgrade the release if already installed UpgradeIfInstalled bool @@ -451,6 +454,7 @@ func generateAction(actionConfig *action.Configuration, cfg ApplyHelmChartConfig case UNINSTALL: return func(c *chart.Chart) error { act := action.NewUninstall(actionConfig) + act.DryRun = cfg.DryRun if _, err := act.Run(cfg.ReleaseName); err != nil { return ErrApplyHelmChart(err) diff --git a/utils/kubernetes/helm.go b/utils/kubernetes/helm.go index 507ed265..d5459eed 100644 --- a/utils/kubernetes/helm.go +++ b/utils/kubernetes/helm.go @@ -23,5 +23,5 @@ func ConvertHelmChartToK8sManifest(cfg ApplyHelmChartConfig) (manifest []byte, e return nil, ErrApplyHelmChart(err) } - return helm.DryRunHelmChart(helmChart) + return helm.DryRunHelmChart(helmChart, cfg.KubernetesVersion) } From 3a32a2a517bdf721eeac8adc863072cbce6cc662 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Fri, 23 Aug 2024 00:27:02 +0530 Subject: [PATCH 058/112] add helper functions for casting Signed-off-by: MUzairS15 --- utils/utils.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/utils/utils.go b/utils/utils.go index 73f1141f..96970d58 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -452,3 +452,46 @@ func FindEntityType(content []byte) (entity.EntityType, error) { } return "", ErrInvalidSchemaVersion } + +// RecursiveCastMapStringInterfaceToMapStringInterface will convert a +// map[string]interface{} recursively => map[string]interface{} +func RecursiveCastMapStringInterfaceToMapStringInterface(in map[string]interface{}) map[string]interface{} { + res := ConvertMapInterfaceMapString(in) + out, ok := res.(map[string]interface{}) + if !ok { + fmt.Println("failed to cast") + } + + return out +} + +// ConvertMapInterfaceMapString converts map[interface{}]interface{} => map[string]interface{} +// +// It will also convert []interface{} => []string +func ConvertMapInterfaceMapString(v interface{}) interface{} { + switch x := v.(type) { + case map[interface{}]interface{}: + m := map[string]interface{}{} + for k, v2 := range x { + switch k2 := k.(type) { + case string: + m[k2] = ConvertMapInterfaceMapString(v2) + default: + m[fmt.Sprint(k)] = ConvertMapInterfaceMapString(v2) + } + } + v = m + + case []interface{}: + for i, v2 := range x { + x[i] = ConvertMapInterfaceMapString(v2) + } + + case map[string]interface{}: + for k, v2 := range x { + x[k] = ConvertMapInterfaceMapString(v2) + } + } + + return v +} From 329e3f65d3bbf924813b0571634335db7c25c13e Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Fri, 23 Aug 2024 21:46:05 +0530 Subject: [PATCH 059/112] update the isnamespace w/signoff Signed-off-by: Jougan-0 --- utils/component/generator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/component/generator.go b/utils/component/generator.go index 417d59cd..288f2780 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -82,9 +82,9 @@ func Generate(crd string) (component.ComponentDefinition, error) { } scope, _ := extractCueValueFromPath(crdCue, DefaultPathConfig.ScopePath) if scope == "Cluster" { - cmp.Metadata.AdditionalProperties["isNamespaced"] = false + cmp.Metadata.IsNamespaced = false } else if scope == "Namespaced" { - cmp.Metadata.AdditionalProperties["isNamespaced"] = true + cmp.Metadata.IsNamespaced = true } cmp.Component.Kind = name if group != "" { From 11ae9662455d2fc11dfd8325b654324a84334a52 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 26 Aug 2024 18:50:56 +0530 Subject: [PATCH 060/112] update schemas to v0.7.26 Signed-off-by: MUzairS15 --- go.mod | 5 +++-- go.sum | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 412607c0..9b091b32 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/google/uuid v1.6.0 github.com/kubernetes/kompose v1.31.1 github.com/layer5io/meshery-operator v0.7.0 - github.com/meshery/schemas v0.7.20 + github.com/meshery/schemas v0.7.26 github.com/nats-io/nats.go v1.31.0 github.com/open-policy-agent/opa v0.67.1 github.com/opencontainers/image-spec v1.1.0 @@ -233,13 +233,14 @@ require ( go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.25.0 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/term v0.22.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/grpc v1.65.0 // indirect google.golang.org/protobuf v1.34.2 // indirect diff --git a/go.sum b/go.sum index 8c11bbdd..02b099de 100644 --- a/go.sum +++ b/go.sum @@ -608,8 +608,8 @@ github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/meshery/kompose v1.0.1 h1:lg8B/pkLh6762jeFsQATD8UJZZwXZf/aviC3/dzw78A= github.com/meshery/kompose v1.0.1/go.mod h1:TWhWTEMbJBUzENf4JTEtBmZRFm/r8n0nS6v4/nSD2vA= -github.com/meshery/schemas v0.7.20 h1:1v5+PKjwq/jq0/dayVNJVDbsrh8GXI5QmDYvEBPDdac= -github.com/meshery/schemas v0.7.20/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= +github.com/meshery/schemas v0.7.26 h1:OK/US9KvcfmERJ8pOiXf1Jw2/nPWrwvZI11U8BcjNVE= +github.com/meshery/schemas v0.7.26/go.mod h1:q/3+ihAvBd0ieuZ5M19MLXsbLeYrSlDxz5TW5zcnpG4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= @@ -927,8 +927,8 @@ golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1107,6 +1107,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= +gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= From bdfb934be0473998e2f2a06f5836306e8cfcf68e Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 27 Aug 2024 19:34:02 +0530 Subject: [PATCH 061/112] support generation of model from openapi schemas Signed-off-by: MUzairS15 --- utils/component/generator.go | 34 +++++++++++++++++++++++++++------- utils/component/utils.go | 11 +---------- utils/kubernetes/crd.go | 20 ++++++++++++++++---- 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/utils/component/generator.go b/utils/component/generator.go index 417d59cd..a0c76f40 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -7,6 +7,7 @@ import ( "cuelang.org/go/cue" "github.com/layer5io/meshkit/utils" + "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1" "github.com/meshery/schemas/models/v1beta1/component" @@ -47,32 +48,43 @@ var DefaultPathConfig2 = CuePathConfig{ var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2} -func Generate(crd string) (component.ComponentDefinition, error) { +func Generate(resource string) (component.ComponentDefinition, error) { cmp := component.ComponentDefinition{} cmp.SchemaVersion = v1beta1.ComponentSchemaVersion cmp.Metadata = component.ComponentDefinition_Metadata{} - crdCue, err := utils.YamlToCue(crd) + isCRD := kubernetes.IsCRD(resource) + + cueValue, err := cueValueFromResource(resource, isCRD) if err != nil { return cmp, err } + + var specPath string + if isCRD { + specPath = DefaultPathConfig.SpecPath + } else { + specPath = "components.schemas" + } + var schema string for _, cfg := range Configs { - schema, err = getSchema(crdCue, cfg) + cfg.SpecPath = specPath + schema, err = getSchema(cueValue, cfg) if err == nil { break } } cmp.Component.Schema = schema - name, err := extractCueValueFromPath(crdCue, DefaultPathConfig.NamePath) + name, err := extractCueValueFromPath(cueValue, DefaultPathConfig.NamePath) if err != nil { return cmp, err } - version, err := extractCueValueFromPath(crdCue, DefaultPathConfig.VersionPath) + version, err := extractCueValueFromPath(cueValue, DefaultPathConfig.VersionPath) if err != nil { return cmp, err } - group, err := extractCueValueFromPath(crdCue, DefaultPathConfig.GroupPath) + group, err := extractCueValueFromPath(cueValue, DefaultPathConfig.GroupPath) if err != nil { return cmp, err } @@ -80,7 +92,7 @@ func Generate(crd string) (component.ComponentDefinition, error) { if cmp.Metadata.AdditionalProperties == nil { cmp.Metadata.AdditionalProperties = make(map[string]interface{}) } - scope, _ := extractCueValueFromPath(crdCue, DefaultPathConfig.ScopePath) + scope, _ := extractCueValueFromPath(cueValue, DefaultPathConfig.ScopePath) if scope == "Cluster" { cmp.Metadata.AdditionalProperties["isNamespaced"] = false } else if scope == "Namespaced" { @@ -98,6 +110,14 @@ func Generate(crd string) (component.ComponentDefinition, error) { return cmp, nil } +func cueValueFromResource(resource string, isCRD bool) (cue.Value, error) { + if isCRD { + return utils.YamlToCue(resource) + } else { + return utils.JsonToCue([]byte(resource)) + } +} + /* Find and modify specific schema properties. 1. Identify interesting properties by walking entire schema. diff --git a/utils/component/utils.go b/utils/component/utils.go index f63d6068..8df80e59 100644 --- a/utils/component/utils.go +++ b/utils/component/utils.go @@ -7,7 +7,6 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" - "gopkg.in/yaml.v2" ) // Remove the fields which is either not required by end user (like status) or is prefilled by system (like apiVersion, kind and metadata) @@ -81,15 +80,7 @@ func FilterCRDs(manifests [][]byte) ([]string, []error) { var errs []error var filteredManifests []string for _, m := range manifests { - - var crd map[string]interface{} - err := yaml.Unmarshal(m, &crd) - if err != nil { - errs = append(errs, err) - continue - } - - isCrd := kubernetes.IsCRD(crd) + isCrd := kubernetes.IsCRD(string(m)) if !isCrd { continue } diff --git a/utils/kubernetes/crd.go b/utils/kubernetes/crd.go index b42cfbc9..cac9d97f 100644 --- a/utils/kubernetes/crd.go +++ b/utils/kubernetes/crd.go @@ -53,7 +53,19 @@ func GetGVRForCustomResources(crd *CRDItem) *schema.GroupVersionResource { } } -func IsCRD(manifest map[string]interface{}) bool { - kind, ok := manifest["kind"].(string) - return ok && kind == "CustomResourceDefinition" -} +func IsCRD(manifest string) bool { + cueValue, err := utils.YamlToCue(manifest) + if err!= nil { + return false + } + kind, err := utils.Lookup(cueValue, "kind") + if err!= nil { + return false + } + kindStr, err := kind.String() + if err != nil { + return false + } + + return kindStr == "CustomResourceDefinition" +} \ No newline at end of file From e3d6d5b9bd96b71175c84d320d9823557378b617 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 28 Aug 2024 18:22:24 +0530 Subject: [PATCH 062/112] update schemas to v0.7.29 Signed-off-by: MUzairS15 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9b091b32..0940a5cf 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/google/uuid v1.6.0 github.com/kubernetes/kompose v1.31.1 github.com/layer5io/meshery-operator v0.7.0 - github.com/meshery/schemas v0.7.26 + github.com/meshery/schemas v0.7.29 github.com/nats-io/nats.go v1.31.0 github.com/open-policy-agent/opa v0.67.1 github.com/opencontainers/image-spec v1.1.0 diff --git a/go.sum b/go.sum index 02b099de..17b46763 100644 --- a/go.sum +++ b/go.sum @@ -608,8 +608,8 @@ github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/meshery/kompose v1.0.1 h1:lg8B/pkLh6762jeFsQATD8UJZZwXZf/aviC3/dzw78A= github.com/meshery/kompose v1.0.1/go.mod h1:TWhWTEMbJBUzENf4JTEtBmZRFm/r8n0nS6v4/nSD2vA= -github.com/meshery/schemas v0.7.26 h1:OK/US9KvcfmERJ8pOiXf1Jw2/nPWrwvZI11U8BcjNVE= -github.com/meshery/schemas v0.7.26/go.mod h1:q/3+ihAvBd0ieuZ5M19MLXsbLeYrSlDxz5TW5zcnpG4= +github.com/meshery/schemas v0.7.29 h1:szMouLFCn/54gllwijaAT4e6VhdoerQwcXssW4VeqRk= +github.com/meshery/schemas v0.7.29/go.mod h1:q/3+ihAvBd0ieuZ5M19MLXsbLeYrSlDxz5TW5zcnpG4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= From 9166b27648166786227f37ac228fe658b1a24c76 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Thu, 29 Aug 2024 19:20:48 +0530 Subject: [PATCH 063/112] update schemas to v0.7.30 Signed-off-by: MUzairS15 --- go.mod | 3 +-- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 0940a5cf..30a9d171 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/google/uuid v1.6.0 github.com/kubernetes/kompose v1.31.1 github.com/layer5io/meshery-operator v0.7.0 - github.com/meshery/schemas v0.7.29 + github.com/meshery/schemas v0.7.30 github.com/nats-io/nats.go v1.31.0 github.com/open-policy-agent/opa v0.67.1 github.com/opencontainers/image-spec v1.1.0 @@ -240,7 +240,6 @@ require ( golang.org/x/term v0.22.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/grpc v1.65.0 // indirect google.golang.org/protobuf v1.34.2 // indirect diff --git a/go.sum b/go.sum index 17b46763..901f769e 100644 --- a/go.sum +++ b/go.sum @@ -608,8 +608,8 @@ github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/meshery/kompose v1.0.1 h1:lg8B/pkLh6762jeFsQATD8UJZZwXZf/aviC3/dzw78A= github.com/meshery/kompose v1.0.1/go.mod h1:TWhWTEMbJBUzENf4JTEtBmZRFm/r8n0nS6v4/nSD2vA= -github.com/meshery/schemas v0.7.29 h1:szMouLFCn/54gllwijaAT4e6VhdoerQwcXssW4VeqRk= -github.com/meshery/schemas v0.7.29/go.mod h1:q/3+ihAvBd0ieuZ5M19MLXsbLeYrSlDxz5TW5zcnpG4= +github.com/meshery/schemas v0.7.30 h1:ILfVqdmqvZD4v+zvcy1sqiYf7ZV6pOLr3vynUIntEkM= +github.com/meshery/schemas v0.7.30/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= @@ -1107,8 +1107,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= -gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= From 996d12641ba3ba58f50b48e6e8690fc1006034c3 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Sat, 31 Aug 2024 18:19:07 +0530 Subject: [PATCH 064/112] openapi-gen Signed-off-by: MUzairS15 --- encoding/convert.go | 15 ++++ generators/github/package.go | 45 +++++----- generators/github/package_test.go | 56 +----------- generators/github/url.go | 6 +- utils/component/generator.go | 21 +++-- utils/component/openapi_generator.go | 123 +++++++++++++++++++++++++++ utils/component/utils.go | 11 +-- utils/helm/helm.go | 11 ++- utils/manifests/utils.go | 15 ++-- 9 files changed, 209 insertions(+), 94 deletions(-) create mode 100644 encoding/convert.go create mode 100644 utils/component/openapi_generator.go diff --git a/encoding/convert.go b/encoding/convert.go new file mode 100644 index 00000000..5dc810eb --- /dev/null +++ b/encoding/convert.go @@ -0,0 +1,15 @@ +package encoding + +import ( + "gopkg.in/yaml.v3" +) + +func ToYaml(data []byte) ([]byte, error) { + var out map[string]interface{} + err := Unmarshal(data, &out) + if err != nil { + return nil, err + } + + return yaml.Marshal(out) +} diff --git a/generators/github/package.go b/generators/github/package.go index 0c632ef9..78ac1ba7 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -2,14 +2,12 @@ package github import ( "bytes" + "fmt" "os" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" - "github.com/layer5io/meshkit/utils/manifests" - "github.com/meshery/schemas/models/v1beta1/category" _component "github.com/meshery/schemas/models/v1beta1/component" - "github.com/meshery/schemas/models/v1beta1/model" ) type GitHubPackage struct { @@ -35,28 +33,33 @@ func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, manifestBytes := bytes.Split(data, []byte("\n---\n")) crds, errs := component.FilterCRDs(manifestBytes) - for _, crd := range crds { - comp, err := component.Generate(crd) + comps, err := component.GenerateFromOpenAPI(crd) if err != nil { - continue - } - if comp.Model.Metadata == nil { - comp.Model.Metadata = &model.ModelDefinition_Metadata{} - } - if comp.Model.Metadata.AdditionalProperties == nil { - comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) + fmt.Println("ERR line 42 : ", err) } - - comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL - comp.Model.Version = gp.version - comp.Model.Name = gp.Name - comp.Model.Category = category.CategoryDefinition{ - Name: "", - } - comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) - components = append(components, comp) + components = append(components, comps...) } + // comp, err := component.Generate(crd) + // if err != nil { + // continue + // } + // if comp.Model.Metadata == nil { + // comp.Model.Metadata = &model.ModelDefinition_Metadata{} + // } + // if comp.Model.Metadata.AdditionalProperties == nil { + // comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) + // } + + // comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL + // comp.Model.Version = gp.version + // comp.Model.Name = gp.Name + // comp.Model.Category = category.CategoryDefinition{ + // Name: "", + // } + // comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) + // components = append(components, comp) + // } return components, utils.CombineErrors(errs, "\n") } diff --git a/generators/github/package_test.go b/generators/github/package_test.go index 14fad58d..21129e87 100644 --- a/generators/github/package_test.go +++ b/generators/github/package_test.go @@ -15,65 +15,13 @@ func TestGenerateCompFromGitHub(t *testing.T) { ghPackageManager GitHubPackageManager want int }{ - // { // Source pointing to a directory - // ghPackageManager: GitHubPackageManager{ - // PackageName: "k8s-config-connector", - // SourceURL: "git://github.com/GoogleCloudPlatform/k8s-config-connector/master/crds/", - // }, - // want: 337, - // }, - // { // Source pointing to a file in a repo - // ghPackageManager: GitHubPackageManager{ - // PackageName: "k8s-config-connector", - // SourceURL: "git://github.com/GoogleCloudPlatform/k8s-config-connector/master/crds/accesscontextmanager_v1alpha1_accesscontextmanageraccesslevelcondition.yaml", - // }, - // want: 1, - // }, - - // { // Source pointing to a directly downloadable file (not a repo per se) - // ghPackageManager: GitHubPackageManager{ - // PackageName: "k8s-config-connector", - // SourceURL: "git://github.com/GoogleCloudPlatform/k8s-config-connector/master/crds/accesscontextmanager_v1alpha1_accesscontextmanageraccesslevelcondition.yaml", - // }, - // want: 1, - // }, - { // Source pointing to a directly downloadable file (not a repo per se) ghPackageManager: GitHubPackageManager{ - PackageName: "k8s-config-connector", - SourceURL: "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-config-connector/master/crds/alloydb_v1beta1_alloydbbackup.yaml/1.113.0", + PackageName: "kuberntes", + SourceURL: "https://raw.githubusercontent.com/kubernetes/kubernetes/master/api/openapi-spec/v3/apis__autoscaling__v2_openapi.json/v1.29.2", }, want: 1, }, - - // { // Source pointing to a directory containing helm chart - // ghPackageManager: GitHubPackageManager{ - // PackageName: "acm-controller", - // SourceURL: "https://meshery.github.io/meshery.io/charts/meshery-v0.7.12.tgz/v0.7.12", - // }, - // want: 2, - // }, - { // Source pointing to a zip containing manifests but no CRDs - ghPackageManager: GitHubPackageManager{ - PackageName: "acm-controller", - SourceURL: "https://github.com/MUzairS15/WASM-filters/raw/main/test.tar.gz/v0.7.12", - }, - want: 0, - }, - { // Source pointing to a zip containing CRDs - ghPackageManager: GitHubPackageManager{ - PackageName: "acm-controller", - SourceURL: "git://github.com/MUzairS15/WASM-filters/main/chart.tar.gz", - }, - want: 14, - }, - // { // Source pointing to a dir containing CRDs - // ghPackageManager: GitHubPackageManager{ - // PackageName: "acm-controller", - // SourceURL: "git://github.com/meshery/meshery/master/install/kubernetes/helm/meshery-operator", - // }, - // want: 2, - // }, } for _, test := range tests { diff --git a/generators/github/url.go b/generators/github/url.go index 6ca4b94d..22a3f1b1 100644 --- a/generators/github/url.go +++ b/generators/github/url.go @@ -2,6 +2,7 @@ package github import ( "bufio" + "fmt" "io" "net/url" @@ -74,10 +75,13 @@ func ProcessContent(w io.Writer, downloadDirPath, downloadfilePath string) error if err != nil { return err - } + } + + fmt.Println("TEST 80 inside github url.go") err = utils.ProcessContent(downloadDirPath, func(path string) error { err = helm.ConvertToK8sManifest(path, "", w) + fmt.Println("TEST 84 inside github url.go") if err != nil { return err } diff --git a/utils/component/generator.go b/utils/component/generator.go index a0c76f40..5987e5e1 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -46,6 +46,16 @@ var DefaultPathConfig2 = CuePathConfig{ SpecPath: "spec.validation.openAPIV3Schema", } +var OpenAPISpecPathConfig = CuePathConfig{ + NamePath: `x-kubernetes-group-version-kind"[0].kind`, + IdentifierPath: "spec.names.kind", + VersionPath: `"x-kubernetes-group-version-kind"[0].version`, + GroupPath: `"x-kubernetes-group-version-kind"[0].group`, + ScopePath: "spec.scope", + SpecPath: "spec.versions[0].schema.openAPIV3Schema", + PropertiesPath: "properties", +} + var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2} func Generate(resource string) (component.ComponentDefinition, error) { @@ -55,11 +65,6 @@ func Generate(resource string) (component.ComponentDefinition, error) { cmp.Metadata = component.ComponentDefinition_Metadata{} isCRD := kubernetes.IsCRD(resource) - cueValue, err := cueValueFromResource(resource, isCRD) - if err != nil { - return cmp, err - } - var specPath string if isCRD { specPath = DefaultPathConfig.SpecPath @@ -67,6 +72,12 @@ func Generate(resource string) (component.ComponentDefinition, error) { specPath = "components.schemas" } + fmt.Println("SPEC PATH ", specPath) + cueValue, err := cueValueFromResource(resource, isCRD) + if err != nil { + return cmp, err + } + var schema string for _, cfg := range Configs { cfg.SpecPath = specPath diff --git a/utils/component/openapi_generator.go b/utils/component/openapi_generator.go new file mode 100644 index 00000000..b340ef65 --- /dev/null +++ b/utils/component/openapi_generator.go @@ -0,0 +1,123 @@ +package component + +import ( + "encoding/json" + "fmt" + "strings" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/cuecontext" + "cuelang.org/go/encoding/yaml" + "github.com/layer5io/meshkit/utils/manifests" + "github.com/meshery/schemas/models/v1beta1" + "github.com/meshery/schemas/models/v1beta1/category" + "github.com/meshery/schemas/models/v1beta1/component" + "github.com/meshery/schemas/models/v1beta1/model" +) + +func GenerateFromOpenAPI(resource string) ([]component.ComponentDefinition, error) { + cuectx := cuecontext.New() + fmt.Println("resource -----------: ", resource) + cueParsedManExpr, err := yaml.Extract("", []byte(resource)) + parsedManifest := cuectx.BuildFile(cueParsedManExpr) + + definitions := parsedManifest.LookupPath(cue.ParsePath("components.schemas")) + if err != nil { + return nil, nil + } + + resol := manifests.ResolveOpenApiRefs{} + cache := make(map[string][]byte) + resolved, err := resol.ResolveReferences([]byte(resource), definitions, cache) + if err != nil { + return nil, err + } + fmt.Println("resource -----------: ", string(resolved)) + cueParsedManExpr, err = yaml.Extract("", []byte(resolved)) + parsedManifest = cuectx.BuildFile(cueParsedManExpr) + definitions = parsedManifest.LookupPath(cue.ParsePath("components.schemas")) + if err != nil { + return nil, nil + } + + fields, err := definitions.Fields() + if err != nil { + fmt.Printf("%v\n", err) + return nil, err + } + components := make([]component.ComponentDefinition, 0) + + for fields.Next() { + fieldVal := fields.Value() + kindCue := fieldVal.LookupPath(cue.ParsePath(`"x-kubernetes-group-version-kind"[0].kind`)) + if kindCue.Err() != nil { + continue + } + kind, err := kindCue.String() + kind = strings.ToLower(kind) + if err != nil { + fmt.Printf("%v", err) + continue + } + + crd, err := fieldVal.MarshalJSON() + if err != nil { + fmt.Printf("%v", err) + continue + } + versionCue := fieldVal.LookupPath(cue.ParsePath(`"x-kubernetes-group-version-kind"[0].version`)) + groupCue := fieldVal.LookupPath(cue.ParsePath(`"x-kubernetes-group-version-kind"[0].group`)) + apiVersion, _ := versionCue.String() + if g, _ := groupCue.String(); g != "" { + apiVersion = g + "/" + apiVersion + } + modified := make(map[string]interface{}) //Remove the given fields which is either not required by End user (like status) or is prefilled by system (like apiVersion, kind and metadata) + err = json.Unmarshal(crd, &modified) + if err != nil { + fmt.Printf("%v", err) + continue + } + + modifiedProps, err := UpdateProperties(fieldVal, cue.ParsePath("properties.spec"), apiVersion) + if err == nil { + modified = modifiedProps + } + + DeleteFields(modified) + crd, err = json.Marshal(modified) + if err != nil { + fmt.Printf("%v", err) + continue + } + + c := component.ComponentDefinition{ + SchemaVersion: v1beta1.ComponentSchemaVersion, + Version: "v1.0.0", + + Format: component.JSON, + Component: component.Component{ + Kind: kind, + Version: apiVersion, + Schema: string(crd), + }, + // Metadata: compMetadata, + DisplayName: manifests.FormatToReadableString(kind), + Model: model.ModelDefinition{ + SchemaVersion: v1beta1.ModelSchemaVersion, + Version: "v1.0.0", + + Model: model.Model{ + Version: "version", + }, + Name: "kubernetes", + DisplayName: "Kubernetes", + Category: category.CategoryDefinition{ + Name: "Orchestration & Management", + }, + }, + } + components = append(components, c) + } + return components, nil + +} diff --git a/utils/component/utils.go b/utils/component/utils.go index 8df80e59..423fbe02 100644 --- a/utils/component/utils.go +++ b/utils/component/utils.go @@ -2,10 +2,10 @@ package component import ( "encoding/json" + "fmt" "cuelang.org/go/cue" "github.com/layer5io/meshkit/utils" - "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" ) @@ -80,10 +80,11 @@ func FilterCRDs(manifests [][]byte) ([]string, []error) { var errs []error var filteredManifests []string for _, m := range manifests { - isCrd := kubernetes.IsCRD(string(m)) - if !isCrd { - continue - } + fmt.Println("TEST 84 : utils.go: ") + // isCrd := kubernetes.IsCRD(string(m)) + // if !isCrd { + + // } filteredManifests = append(filteredManifests, string(m)) } return filteredManifests, errs diff --git a/utils/helm/helm.go b/utils/helm/helm.go index 4af65eed..400450f9 100644 --- a/utils/helm/helm.go +++ b/utils/helm/helm.go @@ -2,6 +2,7 @@ package helm import ( "bytes" + "fmt" "io" "io/fs" "os" @@ -9,6 +10,7 @@ import ( "regexp" "strings" + "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/utils" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" @@ -94,6 +96,7 @@ func ConvertToK8sManifest(path, kubeVersion string, w io.Writer) error { } } else { err := writeToFile(w, path) + fmt.Println("line 98: inside helm.go writeToFile") if err != nil { return err } @@ -108,7 +111,13 @@ func writeToFile(w io.Writer, path string) error { if err != nil { return utils.ErrReadFile(err, path) } - _, err = w.Write(data) + + byt, err := encoding.ToYaml(data) + if err != nil { + return utils.ErrWriteFile(err, path) + } + + _, err = w.Write(byt) if err != nil { return utils.ErrWriteFile(err, path) } diff --git a/utils/manifests/utils.go b/utils/manifests/utils.go index fb3990f7..2935f1c0 100644 --- a/utils/manifests/utils.go +++ b/utils/manifests/utils.go @@ -9,6 +9,7 @@ import ( "strings" "cuelang.org/go/cue" + "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/models/oam/core/v1alpha1" ) @@ -254,7 +255,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue cache = make(map[string][]byte) } var val map[string]interface{} - err := json.Unmarshal(manifest, &val) + err := encoding.Unmarshal(manifest, &val) if err != nil { return nil, err } @@ -266,7 +267,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue if ro.isInsideJsonSchemaProps && (ref == JsonSchemaPropsRef) { // hack so that the UI doesn't crash val["$ref"] = "string" - marVal, errJson := json.Marshal(val) + marVal, errJson := encoding.Marshal(val) if errJson != nil { return manifest, nil } @@ -282,13 +283,13 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue newval = append(newval, v0) continue } - byt, _ := json.Marshal(v0) + byt, _ := encoding.Marshal(v0) byt, err = ro.ResolveReferences(byt, definitions, cache) if err != nil { return nil, err } var newvalmap map[string]interface{} - _ = json.Unmarshal(byt, &newvalmap) + _ = encoding.Unmarshal(byt, &newvalmap) newval = append(newval, newvalmap) } val[k] = newval @@ -333,7 +334,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue if reflect.ValueOf(v).Kind() == reflect.Map { var marVal []byte var def []byte - marVal, err = json.Marshal(v) + marVal, err = encoding.Marshal(v) if err != nil { return nil, err } @@ -349,7 +350,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue } } } - res, err := json.Marshal(val) + res, err := encoding.Marshal(val) if err != nil { return nil, err } @@ -358,7 +359,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue func replaceRefWithVal(def []byte, val map[string]interface{}, k string) error { var defVal map[string]interface{} - err := json.Unmarshal([]byte(def), &defVal) + err := encoding.Unmarshal([]byte(def), &defVal) if err != nil { return err } From 779345898ea1333f7a2da3ad01239e600720d030 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 2 Sep 2024 22:47:46 +0530 Subject: [PATCH 065/112] update schemas package to v0.7.31 Signed-off-by: MUzairS15 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 30a9d171..8a2a8f82 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/google/uuid v1.6.0 github.com/kubernetes/kompose v1.31.1 github.com/layer5io/meshery-operator v0.7.0 - github.com/meshery/schemas v0.7.30 + github.com/meshery/schemas v0.7.31 github.com/nats-io/nats.go v1.31.0 github.com/open-policy-agent/opa v0.67.1 github.com/opencontainers/image-spec v1.1.0 diff --git a/go.sum b/go.sum index 901f769e..0cf6ae53 100644 --- a/go.sum +++ b/go.sum @@ -608,8 +608,8 @@ github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/meshery/kompose v1.0.1 h1:lg8B/pkLh6762jeFsQATD8UJZZwXZf/aviC3/dzw78A= github.com/meshery/kompose v1.0.1/go.mod h1:TWhWTEMbJBUzENf4JTEtBmZRFm/r8n0nS6v4/nSD2vA= -github.com/meshery/schemas v0.7.30 h1:ILfVqdmqvZD4v+zvcy1sqiYf7ZV6pOLr3vynUIntEkM= -github.com/meshery/schemas v0.7.30/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= +github.com/meshery/schemas v0.7.31 h1:FfVR+oErAiiEomt6sTZI5uKhoyU26QXawT6UDZHbthI= +github.com/meshery/schemas v0.7.31/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= From c89de045bdbf887a9d380dce36523be3e37fcf84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 19:53:46 +0000 Subject: [PATCH 066/112] Bump github.com/opencontainers/runc from 1.1.12 to 1.1.14 Bumps [github.com/opencontainers/runc](https://github.com/opencontainers/runc) from 1.1.12 to 1.1.14. - [Release notes](https://github.com/opencontainers/runc/releases) - [Changelog](https://github.com/opencontainers/runc/blob/main/CHANGELOG.md) - [Commits](https://github.com/opencontainers/runc/compare/v1.1.12...v1.1.14) --- updated-dependencies: - dependency-name: github.com/opencontainers/runc dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8a2a8f82..ea10a40a 100644 --- a/go.mod +++ b/go.mod @@ -193,7 +193,7 @@ require ( github.com/novln/docker-parser v1.0.0 // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/runc v1.1.12 // indirect + github.com/opencontainers/runc v1.1.14 // indirect github.com/openshift/api v0.0.0-20200803131051-87466835fcc0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect diff --git a/go.sum b/go.sum index 0cf6ae53..9aad5ca9 100644 --- a/go.sum +++ b/go.sum @@ -707,8 +707,8 @@ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2sz github.com/opencontainers/runc v0.0.0-20161109192122-51371867a01c/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss= -github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= +github.com/opencontainers/runc v1.1.14 h1:rgSuzbmgz5DUJjeSnw337TxDbRuqjs6iqQck/2weR6w= +github.com/opencontainers/runc v1.1.14/go.mod h1:E4C2z+7BxR7GHXp0hAY53mek+x49X1LjPNeMTfRGvOA= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/openshift/api v0.0.0-20200803131051-87466835fcc0 h1:ngLoHyAD7dNUzZY6cBA+X/DWIRLT56n6PjdN9+hqdvs= From 099cdd39516a8e3d2c78f12abb18bb7f78c8fa90 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 9 Sep 2024 18:08:01 +0530 Subject: [PATCH 067/112] update test Signed-off-by: MUzairS15 --- generators/github/package_test.go | 56 +++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/generators/github/package_test.go b/generators/github/package_test.go index 21129e87..14fad58d 100644 --- a/generators/github/package_test.go +++ b/generators/github/package_test.go @@ -15,13 +15,65 @@ func TestGenerateCompFromGitHub(t *testing.T) { ghPackageManager GitHubPackageManager want int }{ + // { // Source pointing to a directory + // ghPackageManager: GitHubPackageManager{ + // PackageName: "k8s-config-connector", + // SourceURL: "git://github.com/GoogleCloudPlatform/k8s-config-connector/master/crds/", + // }, + // want: 337, + // }, + // { // Source pointing to a file in a repo + // ghPackageManager: GitHubPackageManager{ + // PackageName: "k8s-config-connector", + // SourceURL: "git://github.com/GoogleCloudPlatform/k8s-config-connector/master/crds/accesscontextmanager_v1alpha1_accesscontextmanageraccesslevelcondition.yaml", + // }, + // want: 1, + // }, + + // { // Source pointing to a directly downloadable file (not a repo per se) + // ghPackageManager: GitHubPackageManager{ + // PackageName: "k8s-config-connector", + // SourceURL: "git://github.com/GoogleCloudPlatform/k8s-config-connector/master/crds/accesscontextmanager_v1alpha1_accesscontextmanageraccesslevelcondition.yaml", + // }, + // want: 1, + // }, + { // Source pointing to a directly downloadable file (not a repo per se) ghPackageManager: GitHubPackageManager{ - PackageName: "kuberntes", - SourceURL: "https://raw.githubusercontent.com/kubernetes/kubernetes/master/api/openapi-spec/v3/apis__autoscaling__v2_openapi.json/v1.29.2", + PackageName: "k8s-config-connector", + SourceURL: "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-config-connector/master/crds/alloydb_v1beta1_alloydbbackup.yaml/1.113.0", }, want: 1, }, + + // { // Source pointing to a directory containing helm chart + // ghPackageManager: GitHubPackageManager{ + // PackageName: "acm-controller", + // SourceURL: "https://meshery.github.io/meshery.io/charts/meshery-v0.7.12.tgz/v0.7.12", + // }, + // want: 2, + // }, + { // Source pointing to a zip containing manifests but no CRDs + ghPackageManager: GitHubPackageManager{ + PackageName: "acm-controller", + SourceURL: "https://github.com/MUzairS15/WASM-filters/raw/main/test.tar.gz/v0.7.12", + }, + want: 0, + }, + { // Source pointing to a zip containing CRDs + ghPackageManager: GitHubPackageManager{ + PackageName: "acm-controller", + SourceURL: "git://github.com/MUzairS15/WASM-filters/main/chart.tar.gz", + }, + want: 14, + }, + // { // Source pointing to a dir containing CRDs + // ghPackageManager: GitHubPackageManager{ + // PackageName: "acm-controller", + // SourceURL: "git://github.com/meshery/meshery/master/install/kubernetes/helm/meshery-operator", + // }, + // want: 2, + // }, } for _, test := range tests { From c01ff7bec5bc08d831cb38b41d05e1ac0d0997c0 Mon Sep 17 00:00:00 2001 From: Alero Awani Date: Fri, 13 Sep 2024 00:46:18 +0100 Subject: [PATCH 068/112] Allow minify when meshery is installed in-cluster with minikube--author=Alero Awani Signed-off-by: Alero Awani --- utils/kubernetes/client.go | 49 ++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/utils/kubernetes/client.go b/utils/kubernetes/client.go index 146da142..be95809d 100644 --- a/utils/kubernetes/client.go +++ b/utils/kubernetes/client.go @@ -32,40 +32,63 @@ func DetectKubeConfig(configfile []byte) (config *rest.Config, err error) { // Look for kubeconfig from the path mentioned in $KUBECONFIG kubeconfig := os.Getenv("KUBECONFIG") if kubeconfig != "" { - if config, err = clientcmd.BuildConfigFromFlags("", kubeconfig); err == nil { + cfgFile, err := processConfigFromFileLocation(kubeconfig) + if err != nil { + return nil, err + } + if config, err = clientcmd.RESTConfigFromKubeConfig(cfgFile); err == nil { return config, err } } // Look for kubeconfig at the default path path := filepath.Join(utils.GetHome(), ".kube", "config") - if config, err = clientcmd.BuildConfigFromFlags("", path); err == nil { + cfgFile, err := processConfigFromFileLocation(path) + if err != nil { + return nil, err + } + if config, err = clientcmd.RESTConfigFromKubeConfig(cfgFile); err == nil { return config, err } return } -func processConfig(configFile []byte) ([]byte, error) { - cfg, err := clientcmd.Load(configFile) - if err != nil { - return nil, ErrLoadConfig(err) +// processConfigInternal performs the common processing steps on the kubeconfig. +func processConfigInternal(cfg *clientcmdapi.Config) ([]byte, error) { + // Minify the kubeconfig + if err := clientcmdapi.MinifyConfig(cfg); err != nil { + return nil, ErrValidateConfig(err) } - err = clientcmdapi.MinifyConfig(cfg) - if err != nil { + // Flatten the kubeconfig + if err := clientcmdapi.FlattenConfig(cfg); err != nil { return nil, ErrValidateConfig(err) } - err = clientcmdapi.FlattenConfig(cfg) - if err != nil { + // Validate the kubeconfig + if err := clientcmd.Validate(*cfg); err != nil { return nil, ErrValidateConfig(err) } - err = clientcmd.Validate(*cfg) + // Write the processed kubeconfig to a byte slice + return clientcmd.Write(*cfg) +} + +// processConfigFromFileLocation processes the kubeconfig from a file location. +func processConfigFromFileLocation(filename string) ([]byte, error) { + cfg, err := clientcmd.LoadFromFile(filename) if err != nil { - return nil, ErrValidateConfig(err) + return nil, ErrLoadConfig(err) } + return processConfigInternal(cfg) +} - return clientcmd.Write(*cfg) +// processConfig processes the kubeconfig provided as a byte slice. +func processConfig(configFile []byte) ([]byte, error) { + cfg, err := clientcmd.Load(configFile) + if err != nil { + return nil, ErrLoadConfig(err) + } + return processConfigInternal(cfg) } From a6ad2c3e31bd888ddef0e302c912c9ffe6628dba Mon Sep 17 00:00:00 2001 From: Alero Awani Date: Sun, 15 Sep 2024 02:28:59 +0100 Subject: [PATCH 069/112] Create central function in meshkit for processing Kubeconfig files--author=Alero Awani Signed-off-by: Alero Awani --- utils/kubernetes/client.go | 69 +++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/utils/kubernetes/client.go b/utils/kubernetes/client.go index be95809d..9180254e 100644 --- a/utils/kubernetes/client.go +++ b/utils/kubernetes/client.go @@ -14,7 +14,8 @@ import ( func DetectKubeConfig(configfile []byte) (config *rest.Config, err error) { if len(configfile) > 0 { var cfgFile []byte - cfgFile, err = processConfig(configfile) + + _, cfgFile, err = ProcessConfig(configfile, "") if err != nil { return nil, err } @@ -32,7 +33,7 @@ func DetectKubeConfig(configfile []byte) (config *rest.Config, err error) { // Look for kubeconfig from the path mentioned in $KUBECONFIG kubeconfig := os.Getenv("KUBECONFIG") if kubeconfig != "" { - cfgFile, err := processConfigFromFileLocation(kubeconfig) + _, cfgFile, err := ProcessConfig(kubeconfig, "") if err != nil { return nil, err } @@ -43,8 +44,8 @@ func DetectKubeConfig(configfile []byte) (config *rest.Config, err error) { // Look for kubeconfig at the default path path := filepath.Join(utils.GetHome(), ".kube", "config") - cfgFile, err := processConfigFromFileLocation(path) - if err != nil { + _, cfgFile, err := ProcessConfig(path, "") + if err != nil{ return nil, err } if config, err = clientcmd.RESTConfigFromKubeConfig(cfgFile); err == nil { @@ -54,41 +55,49 @@ func DetectKubeConfig(configfile []byte) (config *rest.Config, err error) { return } -// processConfigInternal performs the common processing steps on the kubeconfig. -func processConfigInternal(cfg *clientcmdapi.Config) ([]byte, error) { - // Minify the kubeconfig - if err := clientcmdapi.MinifyConfig(cfg); err != nil { - return nil, ErrValidateConfig(err) +// ProcessConfig handles loading, validating, and optionally saving or returning a kubeconfig +func ProcessConfig(kubeConfig interface{}, outputPath string) (*clientcmdapi.Config, []byte, error) { + var config *clientcmdapi.Config + var err error + + // Load the Kubeconfig + switch v := kubeConfig.(type) { + case string: + config, err = clientcmd.LoadFromFile(v) + case []byte: + config, err = clientcmd.Load(v) + default: + return nil, nil, ErrLoadConfig(err) + } + if err != nil { + return nil, nil, err } - // Flatten the kubeconfig - if err := clientcmdapi.FlattenConfig(cfg); err != nil { - return nil, ErrValidateConfig(err) + // Validate and Process the Config + if err := clientcmdapi.MinifyConfig(config); err != nil { + return nil, nil, ErrValidateConfig(err) } - // Validate the kubeconfig - if err := clientcmd.Validate(*cfg); err != nil { - return nil, ErrValidateConfig(err) + if err := clientcmdapi.FlattenConfig(config); err != nil { + return nil, nil, ErrValidateConfig(err) } - // Write the processed kubeconfig to a byte slice - return clientcmd.Write(*cfg) -} + if err := clientcmd.Validate(*config); err != nil { + return nil, nil, ErrValidateConfig(err) + } -// processConfigFromFileLocation processes the kubeconfig from a file location. -func processConfigFromFileLocation(filename string) ([]byte, error) { - cfg, err := clientcmd.LoadFromFile(filename) + // Convert the config to []byte + configBytes, err := clientcmd.Write(*config) if err != nil { - return nil, ErrLoadConfig(err) + return nil, nil, err } - return processConfigInternal(cfg) -} -// processConfig processes the kubeconfig provided as a byte slice. -func processConfig(configFile []byte) ([]byte, error) { - cfg, err := clientcmd.Load(configFile) - if err != nil { - return nil, ErrLoadConfig(err) + //Save the Processed config to a file + if outputPath != ""{ + if err := clientcmd.WriteToFile(*config, outputPath); err != nil { + return nil, nil, err + } } - return processConfigInternal(cfg) + + return config, configBytes, nil } From e05b6ed2811d4502dff14a0da4c75a316d4d4719 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Mon, 16 Sep 2024 14:17:04 +0530 Subject: [PATCH 070/112] update registration logic to support import command w/signoff Signed-off-by: Jougan-0 --- models/registration/dir.go | 179 +++++++++++++++++++++++++------ models/registration/interface.go | 4 +- models/registration/oci.go | 4 +- models/registration/register.go | 66 +++++++----- models/registration/tar.go | 4 +- utils/utils.go | 38 +++++++ 6 files changed, 232 insertions(+), 63 deletions(-) diff --git a/models/registration/dir.go b/models/registration/dir.go index fb7db466..f283f1e6 100644 --- a/models/registration/dir.go +++ b/models/registration/dir.go @@ -7,7 +7,10 @@ import ( "reflect" "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/oci" + "github.com/layer5io/meshkit/utils" + "github.com/meshery/schemas/models/v1alpha3/relationship" "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/model" @@ -26,69 +29,185 @@ func NewDir(path string) Dir { } /* -PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the packagingUnit struct. +PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the PackagingUnit struct. Invalid definitions are stored in the regErrStore with error data. */ -func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err error) { - pkg := packagingUnit{} - // check if the given is a directory - _, err = os.ReadDir(d.dirpath) +func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ PackagingUnit, err error) { + pkg := PackagingUnit{} + + // Extract the filename to use as entityName in case of errors + filename := filepath.Base(d.dirpath) + + // Check if the given path is accessible + _, err = os.Stat(d.dirpath) if err != nil { - return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not read the directory: %e", err)) + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filename, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not access the path: %w", err))) + return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not access the path: %w", err)) + } + + // Process the path (file or directory) + err = processDir(d.dirpath, &pkg, regErrStore) + if err != nil { + modelName := "" + if !reflect.ValueOf(pkg.Model).IsZero() { + modelName = pkg.Model.Name + } + regErrStore.InsertEntityRegError("", modelName, entity.EntityType("unknown"), filename, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not process the path: %w", err))) + return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not process the path: %w", err)) } - err = filepath.Walk(d.dirpath, func(path string, f os.FileInfo, err error) error { + + if reflect.ValueOf(pkg.Model).IsZero() { + errMsg := fmt.Errorf("model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core must be present, otherwise the model package is considered malformed") + regErrStore.InsertEntityRegError("", "", entity.Model, filename, errMsg) + return pkg, errMsg + } + + return pkg, nil +} + +func processDir(dirPath string, pkg *PackagingUnit, regErrStore RegistrationErrorStore) error { + var tempDirs []string + defer func() { + for _, tempDir := range tempDirs { + os.RemoveAll(tempDir) + } + }() + + return filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { if err != nil { - return err + err = utils.ErrFileWalkDir(fmt.Errorf("error accessing path: %w", err), path) + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + regErrStore.AddInvalidDefinition(path, err) + return nil } - if f.IsDir() { + if info.IsDir() { return nil } - byt, _ := os.ReadFile(path) - if byt == nil { + + // Read the file content + data, err := os.ReadFile(path) + if err != nil { + err = oci.ErrReadingFile(err) + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + regErrStore.AddInvalidDefinition(path, err) return nil } - var e entity.Entity - e, err = getEntity(byt) + // Check if the file is an OCI artifact + if oci.IsOCIArtifact(data) { + // Extract the OCI artifact + tempDir, err := oci.CreateTempOCIContentDir() + if err != nil { + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + regErrStore.AddInvalidDefinition(path, err) + return nil + } + tempDirs = append(tempDirs, tempDir) + err = oci.UnCompressOCIArtifact(path, tempDir) + if err != nil { + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + regErrStore.AddInvalidDefinition(path, err) + return nil + } + // Recursively process the extracted directory + if err := processDir(tempDir, pkg, regErrStore); err != nil { + return err + } + return nil + } + + // Check if the file is a zip or tar file + if utils.IsZip(path) || utils.IsTarGz(path) { + tempDir, err := os.MkdirTemp("", "nested-extract-") + if err != nil { + err = utils.ErrCreateDir(fmt.Errorf("error creating temp directory for nested archive extraction: %w", err), tempDir) + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + regErrStore.AddInvalidDefinition(path, err) + return nil + } + tempDirs = append(tempDirs, tempDir) + if err := utils.ExtractFile(path, tempDir); err != nil { + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + regErrStore.AddInvalidDefinition(path, err) + return nil + } + // Recursively process the extracted directory + if err := processDir(tempDir, pkg, regErrStore); err != nil { + return err + } + return nil + } + + content := data + content, err = utils.YAMLToJSON(content) if err != nil { + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) + return nil + } + // Determine the entity type + entityType, err := utils.FindEntityType(content) + if err != nil { + regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err) regErrStore.AddInvalidDefinition(path, err) return nil } - // set it to pkgunit + if entityType == "" { + // Not an entity we care about + return nil + } + + // Get the entity + var e entity.Entity + e, err = getEntity(content) + if err != nil { + regErrStore.InsertEntityRegError("", "", entityType, filepath.Base(path), fmt.Errorf("could not get entity: %w", err)) + regErrStore.AddInvalidDefinition(path, fmt.Errorf("could not get entity: %w", err)) + return nil + } + + // Add the entity to the packaging unit switch e.Type() { case entity.Model: - if !reflect.ValueOf(pkg.model).IsZero() { - // currently models inside models are not handled - return nil - } model, err := utils.Cast[*model.ModelDefinition](e) if err != nil { + modelName := "" + if model != nil { + modelName = model.Name + } + regErrStore.InsertEntityRegError("", modelName, entityType, modelName, ErrGetEntity(err)) regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) + return nil } - pkg.model = *model + pkg.Model = *model case entity.ComponentDefinition: comp, err := utils.Cast[*component.ComponentDefinition](e) if err != nil { + componentName := "" + if comp != nil { + componentName = comp.Component.Kind + } + regErrStore.InsertEntityRegError("", "", entityType, componentName, ErrGetEntity(err)) regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) + return nil } - pkg.components = append(pkg.components, *comp) + pkg.Components = append(pkg.Components, *comp) case entity.RelationshipDefinition: rel, err := utils.Cast[*relationship.RelationshipDefinition](e) if err != nil { + relationshipName := "" + if rel != nil { + relationshipName = rel.Model.Name + } + regErrStore.InsertEntityRegError("", "", entityType, relationshipName, ErrGetEntity(err)) regErrStore.AddInvalidDefinition(path, ErrGetEntity(err)) + return nil } - pkg.relationships = append(pkg.relationships, *rel) + pkg.Relationships = append(pkg.Relationships, *rel) + default: + // Unhandled entity type + return nil } return nil }) - if err != nil { - return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not completely walk the file tree: %e", err)) - } - if reflect.ValueOf(pkg.model).IsZero() { - err := fmt.Errorf("Model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core....... ...... must be present, otherwise the model package is considered malformed..") - regErrStore.AddInvalidDefinition(d.dirpath, err) - return pkg, err - } - return pkg, err } diff --git a/models/registration/interface.go b/models/registration/interface.go index 3f9f580b..65872e20 100644 --- a/models/registration/interface.go +++ b/models/registration/interface.go @@ -12,11 +12,11 @@ type RegistrationErrorStore interface { InsertEntityRegError(hostname string, modelName string, entityType entity.EntityType, entityName string, err error) } -// Anything that can be parsed into a packagingUnit is a RegisterableEntity in Meshery server +// Anything that can be parsed into a PackagingUnit is a RegisterableEntity in Meshery server type RegisterableEntity interface { /* 1. `err` - this is a breaking error, which signifies that the given entity is invalid and cannot be registered 2. Errors encountered while parsing items into meshmodel entites are stored in the RegistrationErrorStore */ - PkgUnit(RegistrationErrorStore) (packagingUnit, error) + PkgUnit(RegistrationErrorStore) (PackagingUnit, error) } diff --git a/models/registration/oci.go b/models/registration/oci.go index 8c8af21b..02355cf8 100644 --- a/models/registration/oci.go +++ b/models/registration/oci.go @@ -8,7 +8,7 @@ type OCIImage struct { _ gcrv1.Image } -func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error) { - pkg := packagingUnit{} +func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (PackagingUnit, error) { + pkg := PackagingUnit{} return pkg, nil } diff --git a/models/registration/register.go b/models/registration/register.go index 50f86639..e01c969f 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -10,11 +10,11 @@ import ( "github.com/meshery/schemas/models/v1beta1/model" ) -// packaingUnit is the representation of the atomic unit that can be registered into the capabilities registry -type packagingUnit struct { - model model.ModelDefinition - components []component.ComponentDefinition - relationships []relationship.RelationshipDefinition +// PackagingUnit is the representation of the atomic unit that can be registered into the capabilities registry +type PackagingUnit struct { + Model model.ModelDefinition + Components []component.ComponentDefinition + Relationships []relationship.RelationshipDefinition _ []v1beta1.PolicyDefinition } @@ -22,10 +22,11 @@ type RegistrationHelper struct { regManager *meshmodel.RegistryManager regErrStore RegistrationErrorStore svgBaseDir string + PkgUnits []PackagingUnit // Store successfully registered packagingUnits } func NewRegistrationHelper(svgBaseDir string, regm *meshmodel.RegistryManager, regErrStore RegistrationErrorStore) RegistrationHelper { - return RegistrationHelper{svgBaseDir: svgBaseDir, regManager: regm, regErrStore: regErrStore} + return RegistrationHelper{svgBaseDir: svgBaseDir, regManager: regm, regErrStore: regErrStore, PkgUnits: []PackagingUnit{}} } /* @@ -45,11 +46,11 @@ func (rh *RegistrationHelper) Register(entity RegisterableEntity) { register will return an error if it is not able to register the `model`. If there are errors when registering other entities, they are handled properly but does not stop the registration process. */ -func (rh *RegistrationHelper) register(pkg packagingUnit) { +func (rh *RegistrationHelper) register(pkg PackagingUnit) { // 1. Register the model - model := pkg.model + model := pkg.Model - // Dont register anything else if registrant is not there + // Don't register anything else if registrant is not there if model.Registrant.Kind == "" { err := ErrMissingRegistrant(model.Name) rh.regErrStore.InsertEntityRegError(model.Registrant.Kind, "", entity.Model, model.Name, err) @@ -64,24 +65,22 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { var svgCompletePath string - //Write SVG for models - model.Metadata.SvgColor, model.Metadata.SvgWhite, svgCompletePath = WriteAndReplaceSVGWithFileSystemPath(model.Metadata.SvgColor, + // Write SVG for models + model.Metadata.SvgColor, model.Metadata.SvgWhite, svgCompletePath = WriteAndReplaceSVGWithFileSystemPath( + model.Metadata.SvgColor, model.Metadata.SvgWhite, - svgComplete, rh.svgBaseDir, + svgComplete, + rh.svgBaseDir, model.Name, model.Name, ) if svgCompletePath != "" { model.Metadata.SvgComplete = &svgCompletePath } - } model.Registrant.Status = connection.Registered - _, _, err := rh.regManager.RegisterEntity( - model.Registrant, - &model, - ) + _, _, err := rh.regManager.RegisterEntity(model.Registrant, &model) // If model cannot be registered, don't register anything else if err != nil { @@ -91,13 +90,16 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { } hostname := model.Registrant.Kind - modelName := model.Name + + // Prepare slices to hold successfully registered components and relationships + var registeredComponents []component.ComponentDefinition + var registeredRelationships []relationship.RelationshipDefinition // 2. Register components - for _, comp := range pkg.components { + for _, comp := range pkg.Components { comp.Model = model if comp.Styles != nil { - //Write SVG for components + // Write SVG for components comp.Styles.SvgColor, comp.Styles.SvgWhite, comp.Styles.SvgComplete = WriteAndReplaceSVGWithFileSystemPath( comp.Styles.SvgColor, comp.Styles.SvgWhite, @@ -108,23 +110,33 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) { ) } - _, _, err := rh.regManager.RegisterEntity( - model.Registrant, - &comp, - ) + _, _, err := rh.regManager.RegisterEntity(model.Registrant, &comp) if err != nil { err = ErrRegisterEntity(err, string(comp.Type()), comp.DisplayName) - rh.regErrStore.InsertEntityRegError(hostname, modelName, entity.ComponentDefinition, comp.DisplayName, err) + rh.regErrStore.InsertEntityRegError(hostname, model.DisplayName, entity.ComponentDefinition, comp.DisplayName, err) + } else { + // Successful registration, add to successfulComponents + registeredComponents = append(registeredComponents, comp) } } // 3. Register relationships - for _, rel := range pkg.relationships { + for _, rel := range pkg.Relationships { rel.Model = model _, _, err := rh.regManager.RegisterEntity(model.Registrant, &rel) if err != nil { err = ErrRegisterEntity(err, string(rel.Type()), string(rel.Kind)) - rh.regErrStore.InsertEntityRegError(hostname, modelName, entity.RelationshipDefinition, rel.Id.String(), err) + rh.regErrStore.InsertEntityRegError(hostname, model.DisplayName, entity.RelationshipDefinition, rel.Id.String(), err) + } else { + // Successful registration, add to successfulRelationships + registeredRelationships = append(registeredRelationships, rel) } } + + // Update pkg with only successfully registered components and relationships + pkg.Components = registeredComponents + pkg.Relationships = registeredRelationships + pkg.Model = model + // Store the successfully registered PackagingUnit + rh.PkgUnits = append(rh.PkgUnits, pkg) } diff --git a/models/registration/tar.go b/models/registration/tar.go index 139a28bb..3849039d 100644 --- a/models/registration/tar.go +++ b/models/registration/tar.go @@ -4,7 +4,7 @@ type Tar struct { _ string } -func (t Tar) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error) { - pkg := packagingUnit{} +func (t Tar) PkgUnit(regErrStore RegistrationErrorStore) (PackagingUnit, error) { + pkg := PackagingUnit{} return pkg, nil } diff --git a/utils/utils.go b/utils/utils.go index 96970d58..112c3cfe 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -495,3 +495,41 @@ func ConvertMapInterfaceMapString(v interface{}) interface{} { return v } +func ConvertToJSONCompatible(data interface{}) interface{} { + switch v := data.(type) { + case map[interface{}]interface{}: + m := make(map[string]interface{}) + for key, value := range v { + m[key.(string)] = ConvertToJSONCompatible(value) + } + return m + case []interface{}: + for i, item := range v { + v[i] = ConvertToJSONCompatible(item) + } + } + return data +} +func YAMLToJSON(content []byte) ([]byte, error) { + var jsonData interface{} + if err := yaml.Unmarshal(content, &jsonData); err == nil { + jsonData = ConvertToJSONCompatible(jsonData) + convertedContent, err := json.Marshal(jsonData) + if err == nil { + content = convertedContent + } else { + return nil, ErrUnmarshal(err) + } + } else { + return nil, ErrUnmarshal(err) + } + return content, nil +} +func ExtractFile(filePath string, destDir string) error { + if IsTarGz(filePath) { + return ExtractTarGz(destDir, filePath) + } else if IsZip(filePath) { + return ExtractZip(destDir, filePath) + } + return ErrExtractType +} From 8197443946c9e36526d69829d6a85ae5ac9751d8 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Mon, 16 Sep 2024 17:52:34 +0530 Subject: [PATCH 071/112] fix index issue w/signoff Signed-off-by: Jougan-0 --- models/meshmodel/registry/v1beta1/model_filter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index 86971485..fe500864 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -102,7 +102,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e if mf.Greedy { if mf.Id != "" { - finder = finder.First("model_dbs.id = ?", mf.Id) + finder = finder.Where("model_dbs.id = ?", mf.Id) } if mf.Name != "" && mf.DisplayName != "" { finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%") From 1d0652511646c158fb37693d1dbbceeb1f7c1f65 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Mon, 16 Sep 2024 17:57:30 +0530 Subject: [PATCH 072/112] svgdata from path w/signoff Signed-off-by: Jougan-0 --- utils/utils.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/utils/utils.go b/utils/utils.go index 112c3cfe..a65d3438 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -533,3 +533,13 @@ func ExtractFile(filePath string, destDir string) error { } return ErrExtractType } + +// Convert path to svg Data +func ReadSVGData(baseDir, path string) (string, error) { + fullPath := baseDir + path + svgData, err := os.ReadFile(fullPath) + if err != nil { + return "", err + } + return string(svgData), nil +} From d6db6dda600570160f8e5dc2d13966be20605b16 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Wed, 18 Sep 2024 17:40:26 +0530 Subject: [PATCH 073/112] function to compress directory w/signoff Signed-off-by: Jougan-0 --- utils/error.go | 4 ++++ utils/utils.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/utils/error.go b/utils/error.go index 3981a282..b1e50625 100644 --- a/utils/error.go +++ b/utils/error.go @@ -46,6 +46,7 @@ var ( ErrCopyFileCode = "replace_me" ErrCloseFileCode = "replace_me" ErrCompressToTarGZCode = "meshkit-11248" + ErrOpenFileCode = "replace_me" ) var ( ErrExtractType = errors.New( @@ -230,3 +231,6 @@ func ErrCloseFile(err error) error { []string{"Check for issues with file permissions or disk space and try again."}, ) } +func ErrOpenFile(file string) error { + return errors.New(ErrOpenFileCode, errors.Alert, []string{"unable to open file: ", file}, []string{}, []string{"The file does not exist in the location"}, []string{"Make sure to upload the correct file"}) +} diff --git a/utils/utils.go b/utils/utils.go index a65d3438..77b2dd64 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1,7 +1,9 @@ package utils import ( + "archive/tar" "bytes" + "compress/gzip" "encoding/json" "errors" "fmt" @@ -543,3 +545,44 @@ func ReadSVGData(baseDir, path string) (string, error) { } return string(svgData), nil } +func Compress(src string, buf io.Writer) error { + zr := gzip.NewWriter(buf) + defer zr.Close() + tw := tar.NewWriter(zr) + defer tw.Close() + + return filepath.Walk(src, func(file string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + header, err := tar.FileInfoHeader(fi, file) + if err != nil { + return err + } + + relPath, err := filepath.Rel(src, file) + if err != nil { + return err + } + header.Name = filepath.ToSlash(relPath) + + if err := tw.WriteHeader(header); err != nil { + return err + } + + if !fi.IsDir() { + data, err := os.Open(file) + if err != nil { + return err + } + defer data.Close() + + _, err = io.Copy(tw, data) + if err != nil { + return err + } + } + return nil + }) +} From fa51598986d21bcc85a010cf21b62615a1535246 Mon Sep 17 00:00:00 2001 From: Aabid Sofi Date: Thu, 19 Sep 2024 03:19:59 +0530 Subject: [PATCH 074/112] update impport modal schema Signed-off-by: Aabid Sofi --- schemas/configuration/modelImport.json | 81 +++++++++++++++++++++----- 1 file changed, 68 insertions(+), 13 deletions(-) diff --git a/schemas/configuration/modelImport.json b/schemas/configuration/modelImport.json index e3693d40..27fa241f 100644 --- a/schemas/configuration/modelImport.json +++ b/schemas/configuration/modelImport.json @@ -3,10 +3,7 @@ "properties": { "uploadType": { "title": "Upload method", - "enum": [ - "File Upload", - "URL Import" - ], + "enum": ["File Upload", "URL Import"], "default": "Select the Upload Method", "x-rjsf-grid-area": "12", "description": "Choose the method you prefer to upload your model file. Select 'File Upload' if you have the file on your local system or 'URL Import' if you have the file hosted online." @@ -30,9 +27,7 @@ "x-rjsf-grid-area": "12" } }, - "required": [ - "file" - ] + "required": ["file"] } }, { @@ -51,15 +46,75 @@ "title": "URL", "description": "Provide the URL of the design file you want to import. This should be a direct URL to the file, for example: https://raw.github.com/your-design-file.yaml", "x-rjsf-grid-area": "12" + }, + "model": { + "type": "object", + "description": "Provide the details of the model you are uploading incase you are trying to generate a new model", + "properties": { + "modelDisplayName": { + "type": "string" + }, + "registrant": { + "type": "string" + }, + "model": { + "type": "string" + }, + "category": { + "type": "string" + }, + "subCategory": { + "type": "string" + }, + "shape": { + "type": "string" + }, + "primaryColor": { + "type": "string", + "pattern": "^#[0-9A-Fa-f]{6}$" + }, + "secondaryColor": { + "type": "string", + "pattern": "^#[0-9A-Fa-f]{6}$" + }, + "svgColor": { + "type": "string" + }, + "svgWhite": { + "type": "string" + }, + "svgComplete": { + "type": "string" + }, + "isAnnotation": { + "type": "string", + "enum": ["TRUE", "FALSE"] + }, + "publishToRegistry": { + "type": "string", + "enum": ["TRUE", "FALSE"] + } + }, + "required": [ + "modelDisplayName", + "registrant", + "model", + "category", + "subCategory", + "shape", + "primaryColor", + "secondaryColor", + "svgColor", + "svgWhite", + "svgComplete", + "isAnnotation", + "publishToRegistry" + ] } }, - "required": [ - "url" - ] + "required": ["url"] } } ], - "required": [ - "uploadType" - ] + "required": ["uploadType"] } From aedcb7f5911dfe8da92a26c70b013d3f7c8a983a Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Thu, 19 Sep 2024 15:13:14 +0530 Subject: [PATCH 075/112] add support to export in k8s format Signed-off-by: MUzairS15 --- converter/k8s.go | 77 +++++++++++++++++++++++++++++++++++ models/converter/converter.go | 18 ++++++++ models/converter/error.go | 15 +++++++ models/converter/formats.go | 10 +++++ 4 files changed, 120 insertions(+) create mode 100644 converter/k8s.go create mode 100644 models/converter/converter.go create mode 100644 models/converter/error.go create mode 100644 models/converter/formats.go diff --git a/converter/k8s.go b/converter/k8s.go new file mode 100644 index 00000000..7d87c3fb --- /dev/null +++ b/converter/k8s.go @@ -0,0 +1,77 @@ +package converter + +import ( + "bytes" + + "github.com/layer5io/meshkit/models/patterns" + "github.com/layer5io/meshkit/utils" + "github.com/meshery/schemas/models/v1beta1/component" + "github.com/meshery/schemas/models/v1beta1/pattern" + "gopkg.in/yaml.v3" +) + +type K8sConverter struct{} + +func (k *K8sConverter) Convert(patternFile string) (string, error) { + pattern, err := patterns.GetPatternFormat(patternFile) + if err != nil { + return "", err + } + return NewK8sManifestsFromPatternfile(pattern) +} + +func NewK8sManifestsFromPatternfile(patternFile *pattern.PatternFile) (string, error) { + + buf := bytes.NewBufferString("") + + enc := yaml.NewEncoder(buf) + for _, comp := range patternFile.Components { + err := enc.Encode(CreateK8sResourceStructure(comp)) + if err != nil { + return "", err + } + } + return buf.String(), nil +} + +func CreateK8sResourceStructure(comp *component.ComponentDefinition) map[string]interface{} { + annotations := map[string]interface{}{} + labels := map[string]interface{}{} + + _confMetadata, ok := comp.Configuration["metadata"] + if ok { + confMetadata, err := utils.Cast[map[string]interface{}](_confMetadata) + if err == nil { + + _annotations, ok := confMetadata["annotations"] + if ok { + annotations, _ = utils.Cast[map[string]interface{}](_annotations) + } + + _label, ok := confMetadata["labels"] + + if ok { + labels, _ = utils.Cast[map[string]interface{}](_label) + } + } + } + + component := map[string]interface{}{ + "apiVersion": comp.Component.Version, + "kind": comp.Component.Kind, + "metadata": map[string]interface{}{ + "name": comp.DisplayName, + "annotations": annotations, + "labels": labels, + }, + } + + for k, v := range comp.Configuration { + if k == "apiVersion" || k == "kind" || k == "metadata" { + continue + } + + component[k] = v + } + return component +} diff --git a/models/converter/converter.go b/models/converter/converter.go new file mode 100644 index 00000000..3fda332e --- /dev/null +++ b/models/converter/converter.go @@ -0,0 +1,18 @@ +package converter + +import ( + "github.com/layer5io/meshkit/converter" +) + +type ConvertFormat interface { + Convert(string) (string, error) +} + +func NewFormatConverter(format DesignFormat) (ConvertFormat, error) { + switch format { + case K8sManifest: + return &converter.K8sConverter{}, nil + default: + return nil, ErrUnknownFormat(format) + } +} diff --git a/models/converter/error.go b/models/converter/error.go new file mode 100644 index 00000000..6bc9e788 --- /dev/null +++ b/models/converter/error.go @@ -0,0 +1,15 @@ +package converter + +import ( + "fmt" + + "github.com/layer5io/meshkit/errors" +) + +const ( + ErrUnknownFormatCode = "meshkit-11245" +) + +func ErrUnknownFormat(format DesignFormat) error { + return errors.New(ErrUnknownFormatCode, errors.Alert, []string{fmt.Sprintf("\"%s\" format is not supported", format)}, []string{fmt.Sprintf("Failed to export design in \"%s\" format", format)}, []string{"The format is not supported by the current version of Meshery server"}, []string{"Make sure to export design in one of the supported format"}) +} diff --git a/models/converter/formats.go b/models/converter/formats.go new file mode 100644 index 00000000..45c1f090 --- /dev/null +++ b/models/converter/formats.go @@ -0,0 +1,10 @@ +package converter + +type DesignFormat string + +const ( + HelmChart DesignFormat = "Helm Chart" + DockerCompose DesignFormat = "Docker Compose" + K8sManifest DesignFormat = "Kubernetes Manifest" + Design DesignFormat = "Design" +) \ No newline at end of file From 1bb16d72568f083634676fece1bad9b4b23c4f59 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Thu, 19 Sep 2024 15:17:55 +0530 Subject: [PATCH 076/112] fix lint issue Signed-off-by: MUzairS15 --- config/provider/inmem.go | 3 ++- utils/kubernetes/crd.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/provider/inmem.go b/config/provider/inmem.go index 753c0064..0c56f7b5 100644 --- a/config/provider/inmem.go +++ b/config/provider/inmem.go @@ -18,6 +18,7 @@ import ( "sync" "github.com/layer5io/meshkit/config" + "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/utils" ) @@ -54,7 +55,7 @@ func (l *InMem) GetKey(key string) string { func (l *InMem) GetObject(key string, result interface{}) error { l.mutex.Lock() defer l.mutex.Unlock() - return utils.Unmarshal(l.store[key], result) + return encoding.Unmarshal([]byte(l.store[key]), result) } // SetObject sets an object value for the key diff --git a/utils/kubernetes/crd.go b/utils/kubernetes/crd.go index b42cfbc9..0525a624 100644 --- a/utils/kubernetes/crd.go +++ b/utils/kubernetes/crd.go @@ -3,7 +3,7 @@ package kubernetes import ( "context" - "github.com/layer5io/meshkit/utils" + "github.com/layer5io/meshkit/encoding" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" ) @@ -35,7 +35,7 @@ func GetAllCustomResourcesInCluster(ctx context.Context, client rest.Interface) } var xcrd CRD gvks := []*schema.GroupVersionResource{} - err = utils.Unmarshal(string(crdresult), &xcrd) + err = encoding.Unmarshal(crdresult, &xcrd) if err != nil { return nil, err } From ff1a74d51ad3a7a24aee515b8f654cb639aa6cf6 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Thu, 19 Sep 2024 15:20:30 +0530 Subject: [PATCH 077/112] fix lint issue Signed-off-by: MUzairS15 --- validator/validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/validate.go b/validator/validate.go index dc043d52..e91aca19 100644 --- a/validator/validate.go +++ b/validator/validate.go @@ -14,7 +14,7 @@ import ( var ( ErrValidateCode = "" - schemaPath = "components.schemas" + _ = "components.schemas" cueschema cue.Value mx sync.Mutex isSchemaLoaded bool From 41c07af3523562f81914b4e499b9fccc321d9dd5 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Thu, 19 Sep 2024 15:37:21 +0530 Subject: [PATCH 078/112] fix go vet issue Signed-off-by: MUzairS15 --- utils/walker/github.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/walker/github.go b/utils/walker/github.go index 085c0952..bd58eaf5 100644 --- a/utils/walker/github.go +++ b/utils/walker/github.go @@ -7,7 +7,7 @@ import ( "path/filepath" "strings" "sync" - + "errors" "github.com/sirupsen/logrus" ) @@ -164,8 +164,7 @@ func (g *Github) walker(path string, isFile bool) error { if !ok { return fmt.Errorf("[GithubWalker]: GithubAPI responded with: forbidden") } - - return fmt.Errorf(message) + return errors.New(message) } return fmt.Errorf("file not found") } From 09bfdc62d6593d709306e5d4c4700075fb9597b4 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Thu, 19 Sep 2024 20:52:47 +0530 Subject: [PATCH 079/112] update the schema w/signoff Signed-off-by: Jougan-0 --- schemas/configuration/modelImport.json | 125 +++++++++++++++++++++---- 1 file changed, 106 insertions(+), 19 deletions(-) diff --git a/schemas/configuration/modelImport.json b/schemas/configuration/modelImport.json index 27fa241f..fa6b82d6 100644 --- a/schemas/configuration/modelImport.json +++ b/schemas/configuration/modelImport.json @@ -55,19 +55,109 @@ "type": "string" }, "registrant": { - "type": "string" + "type": "string", + "enum": ["github", "artifacthub"] }, "model": { "type": "string" }, "category": { - "type": "string" + "type": "string", + "enum": [ + "Analytics", + "App Definition and Development", + "Cloud Native Network", + "Cloud Native Storage", + "Database", + "Machine Learning", + "Observability and Analysis", + "Orchestration & Management", + "Platform", + "Provisioning", + "Runtime", + "Security & Compliance", + "Serverless", + "Tools", + "Uncategorized" + ] }, "subCategory": { - "type": "string" + "type": "string", + "enum": [ + "Academic", + "API Gateway", + "Application Definition & Image Build", + "Automation & Configuration", + "Certified CNFs", + "Certified Kubernetes - Distribution", + "Certified Kubernetes - Hosted", + "Certified Kubernetes - Installer", + "Chaos Engineering", + "Cloud Native Network", + "Cloud Native Storage", + "Container Registry", + "Container Runtime", + "Continuous Integration & Delivery", + "Continuous Optimization", + "Coordination & Service Discovery", + "Database", + "Debugging and Observability", + "End User Supporter", + "Framework", + "Gold", + "Hosted Platform", + "Installable Platform", + "Key Management", + "Kubernetes Certified Service Provider", + "Kubernetes Training Partner", + "Logging", + "Metrics", + "Monitoring", + "Nonprofit", + "Packaging, Registries & Application Delivery", + "PaaS/Container Service", + "Platinum", + "Remote Procedure Call", + "Runtime", + "Scheduling & Orchestration", + "Security", + "Security & Compliance", + "Service Mesh", + "Service Proxy", + "Silver", + "Specifications", + "Streaming & Messaging", + "Toolchain", + "Tools", + "Tracing" + ] }, "shape": { - "type": "string" + "type": "string", + "enum":[ + "rectangle", + "round-rectangle", + "bottom-round-rectangle", + "cut-rectangle", + "shape", + "circle", + "diamond", + "round-rectang", + "hexagon", + "rhomboid", + "triangle", + "cilinder", + "round-triangle", + "round-pentagon", + "sheild", + "vee", + "cylinder", + "round-heptagon", + "concave-hexagon", + "right-rhomboid", + "barrel", + "round-diamond" + ] }, "primaryColor": { "type": "string", @@ -78,7 +168,8 @@ "pattern": "^#[0-9A-Fa-f]{6}$" }, "svgColor": { - "type": "string" + "type": "string", + "default": "" }, "svgWhite": { "type": "string" @@ -87,28 +178,24 @@ "type": "string" }, "isAnnotation": { - "type": "string", - "enum": ["TRUE", "FALSE"] + "type": "boolean", + "title": "Is Annotation", + "description": "Indicate whether this model is an annotation", + "x-rjsf-grid-area": "6" }, "publishToRegistry": { - "type": "string", - "enum": ["TRUE", "FALSE"] + "type": "boolean", + "title": "Publish to Registry", + "default": true, + "description": "Indicate whether to publish this model to the registry", + "x-rjsf-grid-area": "6" } }, "required": [ "modelDisplayName", "registrant", "model", - "category", - "subCategory", - "shape", - "primaryColor", - "secondaryColor", - "svgColor", - "svgWhite", - "svgComplete", - "isAnnotation", - "publishToRegistry" + "isAnnotation" ] } }, From 5614a67c4aa810172047bdde300120c28e200bf8 Mon Sep 17 00:00:00 2001 From: Alero Awani Date: Fri, 20 Sep 2024 10:51:38 +0100 Subject: [PATCH 080/112] Use Meshkit error--author=Alero Awani Signed-off-by: Alero Awani --- utils/error.go | 5 +++++ utils/kubernetes/client.go | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/utils/error.go b/utils/error.go index 3981a282..7df30d15 100644 --- a/utils/error.go +++ b/utils/error.go @@ -46,6 +46,7 @@ var ( ErrCopyFileCode = "replace_me" ErrCloseFileCode = "replace_me" ErrCompressToTarGZCode = "meshkit-11248" + ErrConvertToByteCode = "meshkit-11187" ) var ( ErrExtractType = errors.New( @@ -150,6 +151,10 @@ func ErrCreateDir(err error, filepath string) error { return errors.New(ErrCreateDirCode, errors.Alert, []string{fmt.Sprintf("error creating directory at %s", filepath)}, []string{err.Error()}, []string{"invalid path provided", "insufficient permissions"}, []string{"provide a valid path", "retry by using an absolute path", "check for sufficient permissions for the user"}) } +func ErrConvertToByte(err error) error { + return errors.New(ErrConvertToByteCode, errors.Alert, []string{("error converting data to []byte")}, []string{err.Error()}, []string{"Unsupported data types", "invalid configuration data", "failed serialization of data"}, []string{"check for any custom types in the data that might not be serializable", "Verify that the data type being passed is valid for conversion to []byte"}) +} + func ErrGettingLatestReleaseTag(err error) error { return errors.New( ErrGettingLatestReleaseTagCode, diff --git a/utils/kubernetes/client.go b/utils/kubernetes/client.go index 9180254e..be5f0e2e 100644 --- a/utils/kubernetes/client.go +++ b/utils/kubernetes/client.go @@ -70,7 +70,7 @@ func ProcessConfig(kubeConfig interface{}, outputPath string) (*clientcmdapi.Con return nil, nil, ErrLoadConfig(err) } if err != nil { - return nil, nil, err + return nil, nil, ErrLoadConfig(err) } // Validate and Process the Config @@ -89,13 +89,13 @@ func ProcessConfig(kubeConfig interface{}, outputPath string) (*clientcmdapi.Con // Convert the config to []byte configBytes, err := clientcmd.Write(*config) if err != nil { - return nil, nil, err + return nil, nil, utils.ErrConvertToByte(err) } //Save the Processed config to a file if outputPath != ""{ if err := clientcmd.WriteToFile(*config, outputPath); err != nil { - return nil, nil, err + return nil, nil, utils.ErrWriteFile(err, outputPath) } } From 6b31b7d922f5405ecf8ef0f6898f7113ed68b1b8 Mon Sep 17 00:00:00 2001 From: Aabid Sofi Date: Fri, 20 Sep 2024 17:15:09 +0530 Subject: [PATCH 081/112] update ui schema Signed-off-by: Aabid Sofi --- .../configuration/uiSchemaModelImport.json | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/schemas/configuration/uiSchemaModelImport.json b/schemas/configuration/uiSchemaModelImport.json index 038796fd..ea24b633 100644 --- a/schemas/configuration/uiSchemaModelImport.json +++ b/schemas/configuration/uiSchemaModelImport.json @@ -1,6 +1,38 @@ { - "uploadType": { - "ui:widget": "radio" + "uploadType": { + "ui:widget": "radio" + }, + "model": { + "ui:options": { + "expand": true }, - "ui:order" : [ "uploadType", "file", "url"] - } + "styles": { + "padding": 0, + "margin": 0 + }, + + "primaryColor": { + "ui:widget": "color" + }, + "secondaryColor": { + "ui:widget": "color" + }, + + "ui:order": [ + "modelDisplayName", + "registrant", + "category", + "subCategory", + "model", + "primaryColor", + "secondaryColor", + "shape", + "svgColor", + "svgComplete", + "svgWhite", + "isAnnotation", + "publishToRegistry" + ] + }, + "ui:order": ["uploadType", "file", "url"] +} From 0072758a1323dcae31b05bed8bc7758ba9dabb43 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Sat, 21 Sep 2024 02:07:06 +0530 Subject: [PATCH 082/112] add missing capabilities w/signoff Signed-off-by: Jougan-0 --- .../registry/v1beta1/model_filter.go | 2 +- schemas/configuration/modelImport.json | 55 ++++++++++--------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index fe500864..6a512f79 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -182,7 +182,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e if includeComponents { var components []component.ComponentDefinition finder := db.Model(&component.ComponentDefinition{}). - Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version,component_definition_dbs.styles"). + Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version,component_definition_dbs.styles,component_definition_dbs.capabilities"). Where("component_definition_dbs.model_id = ?", _modelDB.Id) if err := finder.Scan(&components).Error; err != nil { return nil, 0, 0, err diff --git a/schemas/configuration/modelImport.json b/schemas/configuration/modelImport.json index fa6b82d6..aee0bbc0 100644 --- a/schemas/configuration/modelImport.json +++ b/schemas/configuration/modelImport.json @@ -44,7 +44,7 @@ "type": "string", "format": "uri", "title": "URL", - "description": "Provide the URL of the design file you want to import. This should be a direct URL to the file, for example: https://raw.github.com/your-design-file.yaml", + "description": "Provide the URL of the design file you want to import.", "x-rjsf-grid-area": "12" }, "model": { @@ -52,6 +52,7 @@ "description": "Provide the details of the model you are uploading incase you are trying to generate a new model", "properties": { "modelDisplayName": { + "description":"The model name that would be showed on kanvas and everywhere else mostly of type 'Model Name'", "type": "string" }, "registrant": { @@ -59,27 +60,29 @@ "enum": ["github", "artifacthub"] }, "model": { + "description":"This name is of the type 'model-name'", "type": "string" }, "category": { - "type": "string", - "enum": [ - "Analytics", - "App Definition and Development", - "Cloud Native Network", - "Cloud Native Storage", - "Database", - "Machine Learning", - "Observability and Analysis", - "Orchestration & Management", - "Platform", - "Provisioning", - "Runtime", - "Security & Compliance", - "Serverless", - "Tools", - "Uncategorized" - ] + "description":"The category of the model. The model will be located under the specific category on kanvas.", + "type": "string", + "enum": [ + "Analytics", + "App Definition and Development", + "Cloud Native Network", + "Cloud Native Storage", + "Database", + "Machine Learning", + "Observability and Analysis", + "Orchestration & Management", + "Platform", + "Provisioning", + "Runtime", + "Security & Compliance", + "Serverless", + "Tools", + "Uncategorized" + ] }, "subCategory": { "type": "string", @@ -133,6 +136,7 @@ ] }, "shape": { + "description":"The shape of the model. The model will be displayed in the specific shape on kanvas.", "type": "string", "enum":[ "rectangle", @@ -160,6 +164,7 @@ ] }, "primaryColor": { + "description":"The primary color of the model. This will be the color of the background if the image has one.", "type": "string", "pattern": "^#[0-9A-Fa-f]{6}$" }, @@ -180,22 +185,22 @@ "isAnnotation": { "type": "boolean", "title": "Is Annotation", - "description": "Indicate whether this model is an annotation", + "default": false, + "description": "Indicate whether this model is an annotation. Annotation means that this model is not a real model but an svg image", "x-rjsf-grid-area": "6" }, "publishToRegistry": { "type": "boolean", "title": "Publish to Registry", "default": true, - "description": "Indicate whether to publish this model to the registry", + "description": "Indicate whether to publish this model to the registry. If marked as false then the model won't be registered in the registry", "x-rjsf-grid-area": "6" } }, - "required": [ - "modelDisplayName", + "required": [ "registrant", - "model", - "isAnnotation" + "modelDisplayName", + "model" ] } }, From 6346115126f6fc7d19a48def21a74053146715db Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Sat, 21 Sep 2024 23:11:14 +0530 Subject: [PATCH 083/112] don't register the models that don't have models or relationshisp w/singoff Signed-off-by: Jougan-0 --- models/registration/register.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/models/registration/register.go b/models/registration/register.go index e01c969f..02dde7f5 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -47,6 +47,10 @@ register will return an error if it is not able to register the `model`. If there are errors when registering other entities, they are handled properly but does not stop the registration process. */ func (rh *RegistrationHelper) register(pkg PackagingUnit) { + if len(pkg.Components) == 0 && len(pkg.Relationships) == 0 { + //silently exit if the model does not conatin any components or relationships + return + } // 1. Register the model model := pkg.Model From 368373ee73fb0d2b95ec219136ece2ca7fd0be77 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 23 Sep 2024 12:18:04 +0530 Subject: [PATCH 084/112] update Signed-off-by: MUzairS15 --- generators/github/package.go | 64 +++++++++++++++----------- generators/github/url.go | 6 +-- utils/component/generator.go | 1 - utils/component/openapi_generator.go | 68 ++++++++++++++++++++-------- utils/component/utils.go | 12 ++--- utils/helm/helm.go | 2 - 6 files changed, 93 insertions(+), 60 deletions(-) diff --git a/generators/github/package.go b/generators/github/package.go index 78ac1ba7..7f083bf3 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -2,12 +2,15 @@ package github import ( "bytes" - "fmt" "os" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" + "github.com/layer5io/meshkit/utils/kubernetes" + "github.com/layer5io/meshkit/utils/manifests" + "github.com/meshery/schemas/models/v1beta1/category" _component "github.com/meshery/schemas/models/v1beta1/component" + "github.com/meshery/schemas/models/v1beta1/model" ) type GitHubPackage struct { @@ -32,34 +35,41 @@ func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, } manifestBytes := bytes.Split(data, []byte("\n---\n")) - crds, errs := component.FilterCRDs(manifestBytes) - for _, crd := range crds { - comps, err := component.GenerateFromOpenAPI(crd) - if err != nil { - fmt.Println("ERR line 42 : ", err) + errs := []error{} + + for _, crd := range manifestBytes { + isCrd := kubernetes.IsCRD(string(crd)) + if !isCrd { + + comps, err := component.GenerateFromOpenAPI(string(crd), gp.SourceURL) + if err != nil { + errs = append(errs, err) + continue + } + components = append(components, comps...) + } else { + comp, err := component.Generate(string(crd)) + if err != nil { + continue + } + if comp.Model.Metadata == nil { + comp.Model.Metadata = &model.ModelDefinition_Metadata{} + } + if comp.Model.Metadata.AdditionalProperties == nil { + comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) + } + + comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL + comp.Model.Version = gp.version + comp.Model.Name = gp.Name + comp.Model.Category = category.CategoryDefinition{ + Name: "", + } + comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) + components = append(components, comp) } - components = append(components, comps...) - } - // comp, err := component.Generate(crd) - // if err != nil { - // continue - // } - // if comp.Model.Metadata == nil { - // comp.Model.Metadata = &model.ModelDefinition_Metadata{} - // } - // if comp.Model.Metadata.AdditionalProperties == nil { - // comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) - // } - // comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL - // comp.Model.Version = gp.version - // comp.Model.Name = gp.Name - // comp.Model.Category = category.CategoryDefinition{ - // Name: "", - // } - // comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) - // components = append(components, comp) - // } + } return components, utils.CombineErrors(errs, "\n") } diff --git a/generators/github/url.go b/generators/github/url.go index 22a3f1b1..6ca4b94d 100644 --- a/generators/github/url.go +++ b/generators/github/url.go @@ -2,7 +2,6 @@ package github import ( "bufio" - "fmt" "io" "net/url" @@ -75,13 +74,10 @@ func ProcessContent(w io.Writer, downloadDirPath, downloadfilePath string) error if err != nil { return err - } - - fmt.Println("TEST 80 inside github url.go") + } err = utils.ProcessContent(downloadDirPath, func(path string) error { err = helm.ConvertToK8sManifest(path, "", w) - fmt.Println("TEST 84 inside github url.go") if err != nil { return err } diff --git a/utils/component/generator.go b/utils/component/generator.go index 8dcabe62..3e5af4c9 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -72,7 +72,6 @@ func Generate(resource string) (component.ComponentDefinition, error) { specPath = "components.schemas" } - fmt.Println("SPEC PATH ", specPath) cueValue, err := cueValueFromResource(resource, isCRD) if err != nil { return cmp, err diff --git a/utils/component/openapi_generator.go b/utils/component/openapi_generator.go index b340ef65..25c28de4 100644 --- a/utils/component/openapi_generator.go +++ b/utils/component/openapi_generator.go @@ -7,37 +7,33 @@ import ( "cuelang.org/go/cue" "cuelang.org/go/cue/cuecontext" - "cuelang.org/go/encoding/yaml" + cueJson "cuelang.org/go/encoding/json" + "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/manifests" + + "gopkg.in/yaml.v3" + "github.com/meshery/schemas/models/v1beta1" "github.com/meshery/schemas/models/v1beta1/category" "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/model" ) -func GenerateFromOpenAPI(resource string) ([]component.ComponentDefinition, error) { - cuectx := cuecontext.New() - fmt.Println("resource -----------: ", resource) - cueParsedManExpr, err := yaml.Extract("", []byte(resource)) - parsedManifest := cuectx.BuildFile(cueParsedManExpr) - - definitions := parsedManifest.LookupPath(cue.ParsePath("components.schemas")) - if err != nil { +func GenerateFromOpenAPI(resource, sourceURL string) ([]component.ComponentDefinition, error) { + if resource == "" { return nil, nil } - - resol := manifests.ResolveOpenApiRefs{} - cache := make(map[string][]byte) - resolved, err := resol.ResolveReferences([]byte(resource), definitions, cache) + resource, err := getResolvedManifest(resource) if err != nil { return nil, err } - fmt.Println("resource -----------: ", string(resolved)) - cueParsedManExpr, err = yaml.Extract("", []byte(resolved)) - parsedManifest = cuectx.BuildFile(cueParsedManExpr) - definitions = parsedManifest.LookupPath(cue.ParsePath("components.schemas")) + cuectx := cuecontext.New() + cueParsedManExpr, err := cueJson.Extract("", []byte(resource)) + parsedManifest := cuectx.BuildExpr(cueParsedManExpr) + definitions := parsedManifest.LookupPath(cue.ParsePath("components.schemas")) + if err != nil { - return nil, nil + return nil, err } fields, err := definitions.Fields() @@ -114,10 +110,46 @@ func GenerateFromOpenAPI(resource string) ([]component.ComponentDefinition, erro Category: category.CategoryDefinition{ Name: "Orchestration & Management", }, + Metadata: &model.ModelDefinition_Metadata{ + AdditionalProperties: map[string]interface{}{ + "source_uri": sourceURL, + }, + }, }, } + components = append(components, c) } return components, nil } + +func getResolvedManifest(manifest string) (string, error) { + var m map[string]interface{} + + err := yaml.Unmarshal([]byte(manifest), &m) + if err != nil { + return "", utils.ErrDecodeYaml(err) + } + + byt, err := json.Marshal(m) + if err != nil { + return "", utils.ErrMarshal(err) + } + + cuectx := cuecontext.New() + cueParsedManExpr, err := cueJson.Extract("", byt) + parsedManifest := cuectx.BuildExpr(cueParsedManExpr) + definitions := parsedManifest.LookupPath(cue.ParsePath("components.schemas")) + if err != nil { + return "", err + } + resol := manifests.ResolveOpenApiRefs{} + cache := make(map[string][]byte) + resolved, err := resol.ResolveReferences(byt, definitions, cache) + if err != nil { + return "", err + } + manifest = string(resolved) + return manifest, nil +} diff --git a/utils/component/utils.go b/utils/component/utils.go index 423fbe02..f04ded45 100644 --- a/utils/component/utils.go +++ b/utils/component/utils.go @@ -2,10 +2,10 @@ package component import ( "encoding/json" - "fmt" "cuelang.org/go/cue" "github.com/layer5io/meshkit/utils" + "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" ) @@ -80,12 +80,10 @@ func FilterCRDs(manifests [][]byte) ([]string, []error) { var errs []error var filteredManifests []string for _, m := range manifests { - fmt.Println("TEST 84 : utils.go: ") - // isCrd := kubernetes.IsCRD(string(m)) - // if !isCrd { - - // } - filteredManifests = append(filteredManifests, string(m)) + isCrd := kubernetes.IsCRD(string(m)) + if isCrd { + filteredManifests = append(filteredManifests, string(m)) + } } return filteredManifests, errs } diff --git a/utils/helm/helm.go b/utils/helm/helm.go index 400450f9..091e5902 100644 --- a/utils/helm/helm.go +++ b/utils/helm/helm.go @@ -2,7 +2,6 @@ package helm import ( "bytes" - "fmt" "io" "io/fs" "os" @@ -96,7 +95,6 @@ func ConvertToK8sManifest(path, kubeVersion string, w io.Writer) error { } } else { err := writeToFile(w, path) - fmt.Println("line 98: inside helm.go writeToFile") if err != nil { return err } From 66ee76b4e6bc00a50ef976f1ee379942f2ee00d5 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 23 Sep 2024 22:34:18 +0530 Subject: [PATCH 085/112] remove annotations before converting to manifests Signed-off-by: MUzairS15 --- converter/k8s.go | 2 ++ models/patterns/pattern.go | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/converter/k8s.go b/converter/k8s.go index 7d87c3fb..10c776ca 100644 --- a/converter/k8s.go +++ b/converter/k8s.go @@ -17,6 +17,8 @@ func (k *K8sConverter) Convert(patternFile string) (string, error) { if err != nil { return "", err } + + patterns.ProcessAnnotations(pattern) return NewK8sManifestsFromPatternfile(pattern) } diff --git a/models/patterns/pattern.go b/models/patterns/pattern.go index 5b933132..e206a5c7 100644 --- a/models/patterns/pattern.go +++ b/models/patterns/pattern.go @@ -4,6 +4,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/utils" + "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/pattern" ) @@ -36,3 +37,13 @@ func GetPatternFormat(patternFile string) (*pattern.PatternFile, error) { } return &pattern, nil } + +func ProcessAnnotations(pattern *pattern.PatternFile) { + components := []*component.ComponentDefinition{} + for _, component := range pattern.Components { + if !component.Metadata.IsAnnotation { + components = append(components, component) + } + } + pattern.Components = components +} From 5ee7c9a72660ffe558a0cc50b7ecf6134188310e Mon Sep 17 00:00:00 2001 From: Alero Awani Date: Tue, 24 Sep 2024 09:56:28 +0100 Subject: [PATCH 086/112] add meshkit error for kubeconfig to restconfig--author=Alero Awani Signed-off-by: Alero Awani --- utils/kubernetes/apply-helm-chart.go | 2 +- utils/kubernetes/client.go | 18 +++++++++--------- utils/kubernetes/error.go | 25 +++++++++++++++++++++---- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/utils/kubernetes/apply-helm-chart.go b/utils/kubernetes/apply-helm-chart.go index 7dcc61e9..c633c40f 100644 --- a/utils/kubernetes/apply-helm-chart.go +++ b/utils/kubernetes/apply-helm-chart.go @@ -454,7 +454,7 @@ func generateAction(actionConfig *action.Configuration, cfg ApplyHelmChartConfig case UNINSTALL: return func(c *chart.Chart) error { act := action.NewUninstall(actionConfig) - + act.DryRun = cfg.DryRun if _, err := act.Run(cfg.ReleaseName); err != nil { return ErrApplyHelmChart(err) diff --git a/utils/kubernetes/client.go b/utils/kubernetes/client.go index be5f0e2e..6570fefe 100644 --- a/utils/kubernetes/client.go +++ b/utils/kubernetes/client.go @@ -21,13 +21,13 @@ func DetectKubeConfig(configfile []byte) (config *rest.Config, err error) { } if config, err = clientcmd.RESTConfigFromKubeConfig(cfgFile); err == nil { - return config, err + return config, ErrRestConfigFromKubeConfig(err) } } // If deployed within the cluster if config, err = rest.InClusterConfig(); err == nil { - return config, err + return config, ErrRestConfigFromKubeConfig(err) } // Look for kubeconfig from the path mentioned in $KUBECONFIG @@ -38,18 +38,18 @@ func DetectKubeConfig(configfile []byte) (config *rest.Config, err error) { return nil, err } if config, err = clientcmd.RESTConfigFromKubeConfig(cfgFile); err == nil { - return config, err + return config, ErrRestConfigFromKubeConfig(err) } } // Look for kubeconfig at the default path path := filepath.Join(utils.GetHome(), ".kube", "config") _, cfgFile, err := ProcessConfig(path, "") - if err != nil{ + if err != nil { return nil, err } if config, err = clientcmd.RESTConfigFromKubeConfig(cfgFile); err == nil { - return config, err + return config, ErrRestConfigFromKubeConfig(err) } return @@ -60,7 +60,7 @@ func ProcessConfig(kubeConfig interface{}, outputPath string) (*clientcmdapi.Con var config *clientcmdapi.Config var err error - // Load the Kubeconfig + // Load the Kubeconfig switch v := kubeConfig.(type) { case string: config, err = clientcmd.LoadFromFile(v) @@ -73,7 +73,7 @@ func ProcessConfig(kubeConfig interface{}, outputPath string) (*clientcmdapi.Con return nil, nil, ErrLoadConfig(err) } - // Validate and Process the Config + // Validate and Process the Config if err := clientcmdapi.MinifyConfig(config); err != nil { return nil, nil, ErrValidateConfig(err) } @@ -92,8 +92,8 @@ func ProcessConfig(kubeConfig interface{}, outputPath string) (*clientcmdapi.Con return nil, nil, utils.ErrConvertToByte(err) } - //Save the Processed config to a file - if outputPath != ""{ + //Save the Processed config to a file + if outputPath != "" { if err := clientcmd.WriteToFile(*config, outputPath); err != nil { return nil, nil, utils.ErrWriteFile(err, outputPath) } diff --git a/utils/kubernetes/error.go b/utils/kubernetes/error.go index 10b2d6ea..7d94bfa4 100644 --- a/utils/kubernetes/error.go +++ b/utils/kubernetes/error.go @@ -35,6 +35,7 @@ var ( ErrEntryWithChartVersionNotExistsCode = "meshkit-11204" ErrEndpointNotFound = errors.New(ErrEndpointNotFoundCode, errors.Alert, []string{"Unable to discover an endpoint"}, []string{}, []string{}, []string{}) ErrInvalidAPIServer = errors.New(ErrInvalidAPIServerCode, errors.Alert, []string{"Invalid API Server URL"}, []string{}, []string{}, []string{}) + ErrRestConfigFromKubeConfigCode = "meshkit-11205" ) func ErrApplyManifest(err error) error { @@ -51,22 +52,22 @@ func ErrApplyHelmChart(err error) error { return errors.New(ErrApplyHelmChartCode, errors.Alert, []string{"Error applying helm chart"}, []string{err.Error()}, []string{"Chart could be invalid"}, []string{"Make sure to apply valid chart"}) } -// ErrApplyHelmChart is the error which occurs in the process of applying helm chart +// ErrNewKubeClient is the error which occurs when creating a new Kubernetes clientset func ErrNewKubeClient(err error) error { return errors.New(ErrNewKubeClientCode, errors.Alert, []string{"Error creating kubernetes clientset"}, []string{err.Error()}, []string{"Kubernetes config is not accessible to meshery or not valid"}, []string{"Upload your kubernetes config via the settings dashboard. If uploaded, wait for a minute for it to get initialized"}) } -// ErrApplyHelmChart is the error which occurs in the process of applying helm chart +// ErrNewDynClient is the error which occurs when creating a new dynamic client func ErrNewDynClient(err error) error { return errors.New(ErrNewDynClientCode, errors.Alert, []string{"Error creating dynamic client"}, []string{err.Error()}, []string{"Kubernetes config is not accessible to meshery or not valid"}, []string{"Upload your kubernetes config via the settings dashboard. If uploaded, wait for a minute for it to get initialized"}) } -// ErrApplyHelmChart is the error which occurs in the process of applying helm chart +// ErrNewDiscovery is the error which occurs when creating a new discovery client func ErrNewDiscovery(err error) error { return errors.New(ErrNewDiscoveryCode, errors.Alert, []string{"Error creating discovery client"}, []string{err.Error()}, []string{"Discovery resource is invalid or doesnt exist"}, []string{"Makes sure the you input valid resource for discovery"}) } -// ErrApplyHelmChart is the error which occurs in the process of applying helm chart +// ErrNewInformer is the error which occurs when creating a new informer func ErrNewInformer(err error) error { return errors.New(ErrNewInformerCode, errors.Alert, []string{"Error creating informer client"}, []string{err.Error()}, []string{"Informer is invalid or doesnt exist"}, []string{"Makes sure the you input valid resource for the informer"}) } @@ -100,3 +101,19 @@ func ErrEntryWithChartVersionNotExists(entry, appVersion string) error { func ErrHelmRepositoryNotFound(repo string, err error) error { return errors.New(ErrHelmRepositoryNotFoundCode, errors.Alert, []string{"Helm repo not found"}, []string{fmt.Sprintf("either the repo %s does not exists or is corrupt: %v", repo, err)}, []string{}, []string{}) } + +// ErrRestConfigFromKubeConfig returns an error when failing to create a REST config from a kubeconfig file. +func ErrRestConfigFromKubeConfig(err error) error { + return errors.New(ErrRestConfigFromKubeConfigCode, + errors.Alert, + []string{"Failed to create REST config from kubeconfig."}, + []string{fmt.Sprintf("Error occured while creating REST config from kubeconfig: %s", err.Error())}, + []string{ + "The provided kubeconfig data might be invalid or corrupted.", + "The kubeconfig might be incomplete or missing required fields."}, + []string{ + "Verify that the kubeconfig data is valid.", + "Ensure the kubeconfig contains all necessary cluster, user, and context information.", + "Check if the kubeconfig data was properly read and passed to the function."}, + ) +} From 5ef3a09b1f15c71bac01238013495d90dd091958 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Wed, 25 Sep 2024 01:06:10 +0530 Subject: [PATCH 087/112] update schema for commit w/signoff Signed-off-by: Jougan-0 --- schemas/configuration/generate.json | 223 ++++++++++++++++++ schemas/configuration/modelImport.json | 168 +------------ schemas/configuration/uiSchemaGenerate.json | 38 +++ .../configuration/uiSchemaModelImport.json | 32 --- schemas/schemaProvider.go | 2 + 5 files changed, 269 insertions(+), 194 deletions(-) create mode 100644 schemas/configuration/generate.json create mode 100644 schemas/configuration/uiSchemaGenerate.json diff --git a/schemas/configuration/generate.json b/schemas/configuration/generate.json new file mode 100644 index 00000000..8221595b --- /dev/null +++ b/schemas/configuration/generate.json @@ -0,0 +1,223 @@ +{ + "type": "object", + "properties": { + "uploadType": { + "title": "Upload method", + "enum": ["URL Import", "CSV Import"], + "default": "Select the Upload Method", + "x-rjsf-grid-area": "6", + "description": "Choose the method you prefer to upload your model file. Select 'File Upload' if you have the file on your local system or 'URL Import' if you have the file hosted online." + }, + "register": { + "type": "boolean", + "title": "Register", + "default": true, + "description": "Indicate whether to register this model.", + "x-rjsf-grid-area": "6" + } + }, + "allOf": [ + { + "if": { + "properties": { + "uploadType": { + "const": "CSV Import" + } + } + }, + "then": { + "properties": { + "model_csv": { + "type": "string", + "format": "file", + "description": "Provide the model CSV here. Example template can be seen on [docs.meshery.io](https://docs.meshery.io)", + "x-rjsf-grid-area": "6" + }, + "component_csv": { + "type": "string", + "format": "file", + "description": "Browse the filter file from your file system", + "x-rjsf-grid-area": "6" + }, + "relationship": { + "type": "string", + "format": "file", + "description": "Browse the filter file from your file system", + "x-rjsf-grid-area": "6" + } + }, + "required": ["model_csv", "component_csv"] + } + }, + { + "if": { + "properties": { + "uploadType": { + "const": "URL Import" + } + } + }, + "then": { + "properties": { + "url": { + "type": "string", + "format": "uri", + "title": "URL", + "description": "Provide the URL of the design file you want to import.", + "x-rjsf-grid-area": "12" + }, + "model": { + "type": "object", + "description": "Provide the details of the model you are uploading incase you are trying to generate a new model", + "properties": { + "modelDisplayName": { + "description": "The model name that would be showed on kanvas and everywhere else mostly of type 'Model Name'", + "type": "string" + }, + "registrant": { + "title": "Source", + "type": "string", + "enum": ["github", "artifacthub"] + }, + "model": { + "description": "This name is of the type 'model-name'", + "type": "string" + }, + "category": { + "description": "The category of the model. The model will be located under the specific category on kanvas.", + "type": "string", + "enum": [ + "Analytics", + "App Definition and Development", + "Cloud Native Network", + "Cloud Native Storage", + "Database", + "Machine Learning", + "Observability and Analysis", + "Orchestration & Management", + "Platform", + "Provisioning", + "Runtime", + "Security & Compliance", + "Serverless", + "Tools", + "Uncategorized" + ] + }, + "subCategory": { + "type": "string", + "enum": [ + "Academic", + "API Gateway", + "Application Definition & Image Build", + "Automation & Configuration", + "Certified CNFs", + "Certified Kubernetes - Distribution", + "Certified Kubernetes - Hosted", + "Certified Kubernetes - Installer", + "Chaos Engineering", + "Cloud Native Network", + "Cloud Native Storage", + "Container Registry", + "Container Runtime", + "Continuous Integration & Delivery", + "Continuous Optimization", + "Coordination & Service Discovery", + "Database", + "Debugging and Observability", + "End User Supporter", + "Framework", + "Gold", + "Hosted Platform", + "Installable Platform", + "Key Management", + "Kubernetes Certified Service Provider", + "Kubernetes Training Partner", + "Logging", + "Metrics", + "Monitoring", + "Nonprofit", + "Packaging, Registries & Application Delivery", + "PaaS/Container Service", + "Platinum", + "Remote Procedure Call", + "Runtime", + "Scheduling & Orchestration", + "Security", + "Security & Compliance", + "Service Mesh", + "Service Proxy", + "Silver", + "Specifications", + "Streaming & Messaging", + "Toolchain", + "Tools", + "Tracing" + ] + }, + "shape": { + "description": "The shape of the model. The model will be displayed in the specific shape on kanvas.", + "type": "string", + "enum": [ + "rectangle", + "round-rectangle", + "bottom-round-rectangle", + "cut-rectangle", + "shape", + "circle", + "diamond", + "round-rectang", + "hexagon", + "rhomboid", + "triangle", + "cilinder", + "round-triangle", + "round-pentagon", + "sheild", + "vee", + "cylinder", + "round-heptagon", + "concave-hexagon", + "right-rhomboid", + "barrel", + "round-diamond" + ] + }, + "primaryColor": { + "description": "The primary color of the model. This will be the color of the background if the image has one.", + "type": "string", + "pattern": "^#[0-9A-Fa-f]{6}$" + }, + "secondaryColor": { + "type": "string", + "pattern": "^#[0-9A-Fa-f]{6}$" + }, + "svgColor": { + "type": "string", + "default": "" + }, + "svgWhite": { + "type": "string" + }, + "isAnnotation": { + "type": "boolean", + "title": "Visual Annotation", + "default": false, + "description": "Indicate whether this model is an annotation. Annotation means that this model is not a real model but an svg image", + "x-rjsf-grid-area": "6" + } + }, + "required": [ + "registrant", + "modelDisplayName", + "model" + ] + } + }, + "required": ["url"] + } + } + ], + "required": ["uploadType", "register"] + } + \ No newline at end of file diff --git a/schemas/configuration/modelImport.json b/schemas/configuration/modelImport.json index aee0bbc0..61f1098a 100644 --- a/schemas/configuration/modelImport.json +++ b/schemas/configuration/modelImport.json @@ -3,7 +3,7 @@ "properties": { "uploadType": { "title": "Upload method", - "enum": ["File Upload", "URL Import"], + "enum": ["File Import", "URL Import"], "default": "Select the Upload Method", "x-rjsf-grid-area": "12", "description": "Choose the method you prefer to upload your model file. Select 'File Upload' if you have the file on your local system or 'URL Import' if you have the file hosted online." @@ -14,7 +14,7 @@ "if": { "properties": { "uploadType": { - "const": "File Upload" + "const": "File Import" } } }, @@ -44,167 +44,11 @@ "type": "string", "format": "uri", "title": "URL", - "description": "Provide the URL of the design file you want to import.", - "x-rjsf-grid-area": "12" - }, - "model": { - "type": "object", - "description": "Provide the details of the model you are uploading incase you are trying to generate a new model", - "properties": { - "modelDisplayName": { - "description":"The model name that would be showed on kanvas and everywhere else mostly of type 'Model Name'", - "type": "string" - }, - "registrant": { - "type": "string", - "enum": ["github", "artifacthub"] - }, - "model": { - "description":"This name is of the type 'model-name'", - "type": "string" - }, - "category": { - "description":"The category of the model. The model will be located under the specific category on kanvas.", - "type": "string", - "enum": [ - "Analytics", - "App Definition and Development", - "Cloud Native Network", - "Cloud Native Storage", - "Database", - "Machine Learning", - "Observability and Analysis", - "Orchestration & Management", - "Platform", - "Provisioning", - "Runtime", - "Security & Compliance", - "Serverless", - "Tools", - "Uncategorized" - ] - }, - "subCategory": { - "type": "string", - "enum": [ - "Academic", - "API Gateway", - "Application Definition & Image Build", - "Automation & Configuration", - "Certified CNFs", - "Certified Kubernetes - Distribution", - "Certified Kubernetes - Hosted", - "Certified Kubernetes - Installer", - "Chaos Engineering", - "Cloud Native Network", - "Cloud Native Storage", - "Container Registry", - "Container Runtime", - "Continuous Integration & Delivery", - "Continuous Optimization", - "Coordination & Service Discovery", - "Database", - "Debugging and Observability", - "End User Supporter", - "Framework", - "Gold", - "Hosted Platform", - "Installable Platform", - "Key Management", - "Kubernetes Certified Service Provider", - "Kubernetes Training Partner", - "Logging", - "Metrics", - "Monitoring", - "Nonprofit", - "Packaging, Registries & Application Delivery", - "PaaS/Container Service", - "Platinum", - "Remote Procedure Call", - "Runtime", - "Scheduling & Orchestration", - "Security", - "Security & Compliance", - "Service Mesh", - "Service Proxy", - "Silver", - "Specifications", - "Streaming & Messaging", - "Toolchain", - "Tools", - "Tracing" - ] - }, - "shape": { - "description":"The shape of the model. The model will be displayed in the specific shape on kanvas.", - "type": "string", - "enum":[ - "rectangle", - "round-rectangle", - "bottom-round-rectangle", - "cut-rectangle", - "shape", - "circle", - "diamond", - "round-rectang", - "hexagon", - "rhomboid", - "triangle", - "cilinder", - "round-triangle", - "round-pentagon", - "sheild", - "vee", - "cylinder", - "round-heptagon", - "concave-hexagon", - "right-rhomboid", - "barrel", - "round-diamond" - ] - }, - "primaryColor": { - "description":"The primary color of the model. This will be the color of the background if the image has one.", - "type": "string", - "pattern": "^#[0-9A-Fa-f]{6}$" - }, - "secondaryColor": { - "type": "string", - "pattern": "^#[0-9A-Fa-f]{6}$" - }, - "svgColor": { - "type": "string", - "default": "" - }, - "svgWhite": { - "type": "string" - }, - "svgComplete": { - "type": "string" - }, - "isAnnotation": { - "type": "boolean", - "title": "Is Annotation", - "default": false, - "description": "Indicate whether this model is an annotation. Annotation means that this model is not a real model but an svg image", - "x-rjsf-grid-area": "6" - }, - "publishToRegistry": { - "type": "boolean", - "title": "Publish to Registry", - "default": true, - "description": "Indicate whether to publish this model to the registry. If marked as false then the model won't be registered in the registry", - "x-rjsf-grid-area": "6" - } - }, - "required": [ - "registrant", - "modelDisplayName", - "model" - ] + "description": "This feature is upcoming and currently unavailable.", + "x-rjsf-grid-area": "12", + "disabled": true } - }, - "required": ["url"] + } } } ], diff --git a/schemas/configuration/uiSchemaGenerate.json b/schemas/configuration/uiSchemaGenerate.json new file mode 100644 index 00000000..e3197039 --- /dev/null +++ b/schemas/configuration/uiSchemaGenerate.json @@ -0,0 +1,38 @@ +{ + "uploadType": { + "ui:widget": "radio" + }, + "model": { + "ui:options": { + "expand": true + }, + "styles": { + "padding": 0, + "margin": 0 + }, + + "primaryColor": { + "ui:widget": "color" + }, + "secondaryColor": { + "ui:widget": "color" + }, + + "ui:order": [ + "modelDisplayName", + "model", + "registrant", + "shape", + "category", + "subCategory", + "primaryColor", + "secondaryColor", + "svgColor", + "svgWhite", + "isAnnotation", + "publishToRegistry" + ] + }, + "ui:order": ["uploadType","register","url"] + } + \ No newline at end of file diff --git a/schemas/configuration/uiSchemaModelImport.json b/schemas/configuration/uiSchemaModelImport.json index ea24b633..7ad06933 100644 --- a/schemas/configuration/uiSchemaModelImport.json +++ b/schemas/configuration/uiSchemaModelImport.json @@ -2,37 +2,5 @@ "uploadType": { "ui:widget": "radio" }, - "model": { - "ui:options": { - "expand": true - }, - "styles": { - "padding": 0, - "margin": 0 - }, - - "primaryColor": { - "ui:widget": "color" - }, - "secondaryColor": { - "ui:widget": "color" - }, - - "ui:order": [ - "modelDisplayName", - "registrant", - "category", - "subCategory", - "model", - "primaryColor", - "secondaryColor", - "shape", - "svgColor", - "svgComplete", - "svgWhite", - "isAnnotation", - "publishToRegistry" - ] - }, "ui:order": ["uploadType", "file", "url"] } diff --git a/schemas/schemaProvider.go b/schemas/schemaProvider.go index 8e871e68..6e809e4e 100644 --- a/schemas/schemaProvider.go +++ b/schemas/schemaProvider.go @@ -14,6 +14,7 @@ func getSchemaMap() map[string]string { "environment": "configuration/environment.json", "workspace": "configuration/workspace.json", "model": "configuration/modelImport.json", + "generate": "configuration/generate.json", } } @@ -27,6 +28,7 @@ func getUiSchemaMap() map[string]string { "environment": "configuration/uiSchemaEnvironment.json", "workspace": "configuration/uiSchemaWorkspace.json", "model": "configuration/uiSchemaModelImport.json", + "generate": "configuration/uiSchemaGenerate.json", } } From 5ba263e6339d365e307096fbc1d18cb4770e3579 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 27 Aug 2024 19:34:02 +0530 Subject: [PATCH 088/112] support generation of model from openapi schemas Signed-off-by: MUzairS15 --- utils/component/generator.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils/component/generator.go b/utils/component/generator.go index 3e5af4c9..0b974245 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -65,6 +65,11 @@ func Generate(resource string) (component.ComponentDefinition, error) { cmp.Metadata = component.ComponentDefinition_Metadata{} isCRD := kubernetes.IsCRD(resource) + cueValue, err := cueValueFromResource(resource, isCRD) + if err != nil { + return cmp, err + } + var specPath string if isCRD { specPath = DefaultPathConfig.SpecPath @@ -72,11 +77,6 @@ func Generate(resource string) (component.ComponentDefinition, error) { specPath = "components.schemas" } - cueValue, err := cueValueFromResource(resource, isCRD) - if err != nil { - return cmp, err - } - var schema string for _, cfg := range Configs { cfg.SpecPath = specPath From b8eebd9ed11df5ef542f5d1cd7ae78c315297a21 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 23 Sep 2024 22:28:50 +0530 Subject: [PATCH 089/112] temp Signed-off-by: MUzairS15 --- .../github/kubernetes/apiresourcelist.json | 57 +++++++++++++++++++ .../github/kubernetes/deleteoptions.json | 57 +++++++++++++++++++ .../kubernetes/horizontalpodautoscaler.json | 57 +++++++++++++++++++ .../horizontalpodautoscalerlist.json | 57 +++++++++++++++++++ generators/github/kubernetes/status.json | 57 +++++++++++++++++++ generators/github/kubernetes/watchevent.json | 57 +++++++++++++++++++ 6 files changed, 342 insertions(+) create mode 100644 generators/github/kubernetes/apiresourcelist.json create mode 100644 generators/github/kubernetes/deleteoptions.json create mode 100644 generators/github/kubernetes/horizontalpodautoscaler.json create mode 100644 generators/github/kubernetes/horizontalpodautoscalerlist.json create mode 100644 generators/github/kubernetes/status.json create mode 100644 generators/github/kubernetes/watchevent.json diff --git a/generators/github/kubernetes/apiresourcelist.json b/generators/github/kubernetes/apiresourcelist.json new file mode 100644 index 00000000..8f9d8b74 --- /dev/null +++ b/generators/github/kubernetes/apiresourcelist.json @@ -0,0 +1,57 @@ +{ +"component": { +"kind": "apiresourcelist", +"schema": "{\"description\":\"APIResourceList is a list of APIResource, it is used to expose the name of the resources supported in a specific group and version, and if the resource is namespaced.\",\"properties\":{\"groupVersion\":{\"default\":\"\",\"description\":\"groupVersion is the group and version this APIResourceList is for.\",\"type\":\"string\"},\"resources\":{\"description\":\"resources contains the name of the resources and if they are namespaced.\",\"items\":{\"allOf\":[{\"description\":\"APIResource specifies the name of a resource and whether it is namespaced.\",\"properties\":{\"categories\":{\"description\":\"categories is a list of the grouped resources this resource belongs to (e.g. 'all')\",\"items\":{\"default\":\"\",\"type\":\"string\"},\"type\":\"array\",\"x-kubernetes-list-type\":\"atomic\"},\"group\":{\"description\":\"group is the preferred group of the resource. Empty implies the group of the containing resource list. For subresources, this may have a different value, for example: Scale\\\".\",\"type\":\"string\"},\"kind\":{\"default\":\"\",\"description\":\"kind is the kind for the resource (e.g. 'Foo' is the kind for a resource 'foo')\",\"type\":\"string\"},\"name\":{\"default\":\"\",\"description\":\"name is the plural name of the resource.\",\"type\":\"string\"},\"namespaced\":{\"default\":false,\"description\":\"namespaced indicates if a resource is namespaced or not.\",\"type\":\"boolean\"},\"shortNames\":{\"description\":\"shortNames is a list of suggested short names of the resource.\",\"items\":{\"default\":\"\",\"type\":\"string\"},\"type\":\"array\",\"x-kubernetes-list-type\":\"atomic\"},\"singularName\":{\"default\":\"\",\"description\":\"singularName is the singular name of the resource. This allows clients to handle plural and singular opaquely. The singularName is more correct for reporting status on a single item and both singular and plural are allowed from the kubectl CLI interface.\",\"type\":\"string\"},\"storageVersionHash\":{\"description\":\"The hash value of the storage version, the version this resource is converted to when written to the data store. Value must be treated as opaque by clients. Only equality comparison on the value is valid. This is an alpha feature and may change or be removed in the future. The field is populated by the apiserver only if the StorageVersionHash feature gate is enabled. This field will remain optional even if it graduates.\",\"type\":\"string\"},\"verbs\":{\"description\":\"verbs is a list of supported kube verbs (this includes get, list, watch, create, update, patch, delete, deletecollection, and proxy)\",\"items\":{\"default\":\"\",\"type\":\"string\"},\"type\":\"array\"},\"version\":{\"description\":\"version is the preferred version of the resource. Empty implies the version of the containing resource list For subresources, this may have a different value, for example: v1 (while inside a v1beta1 version of the core resource's group)\\\".\",\"type\":\"string\"}},\"required\":[\"name\",\"singularName\",\"namespaced\",\"kind\",\"verbs\"],\"type\":\"object\"}],\"default\":{}},\"type\":\"array\",\"x-kubernetes-list-type\":\"atomic\"}},\"required\":[\"groupVersion\",\"resources\"],\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"\",\"kind\":\"APIResourceList\",\"version\":\"v1\"}]}", +"version": "v1" +}, +"configuration": null, +"description": "", +"displayName": "apiresourcelist", +"format": "JSON", +"id": "00000000-0000-0000-0000-000000000000", +"metadata": { +"genealogy": "", +"isAnnotation": false, +"isNamespaced": false, +"published": false +}, +"model": { +"category": { +"name": "Orchestration \u0026 Management" +}, +"displayName": "Kubernetes", +"id": "00000000-0000-0000-0000-000000000000", +"metadata": { +"source_uri": "https://raw.githubusercontent.com/kubernetes/kubernetes/refs/heads/master/api/openapi-spec/v3/apis__autoscaling__v1_openapi.json/v1.31.3", +"svgColor": "", +"svgWhite": "" +}, +"model": { +"version": "version" +}, +"name": "kubernetes", +"registrant": { +"created_at": "0001-01-01T00:00:00Z", +"credential_id": "00000000-0000-0000-0000-000000000000", +"deleted_at": "0001-01-01T00:00:00Z", +"id": "00000000-0000-0000-0000-000000000000", +"kind": "", +"name": "", +"status": "", +"sub_type": "", +"type": "", +"updated_at": "0001-01-01T00:00:00Z", +"user_id": "00000000-0000-0000-0000-000000000000" +}, +"connection_id": "00000000-0000-0000-0000-000000000000", +"schemaVersion": "models.meshery.io/v1beta1", +"status": "", +"version": "v1.0.0", +"components": null, +"relationships": null +}, +"schemaVersion": "components.meshery.io/v1beta1", +"status": null, +"styles": null, +"version": "v1.0.0" +} \ No newline at end of file diff --git a/generators/github/kubernetes/deleteoptions.json b/generators/github/kubernetes/deleteoptions.json new file mode 100644 index 00000000..41ebe8d6 --- /dev/null +++ b/generators/github/kubernetes/deleteoptions.json @@ -0,0 +1,57 @@ +{ +"component": { +"kind": "deleteoptions", +"schema": "{\"description\":\"DeleteOptions may be provided when deleting an API object.\",\"properties\":{\"dryRun\":{\"description\":\"When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed\",\"items\":{\"default\":\"\",\"type\":\"string\"},\"type\":\"array\",\"x-kubernetes-list-type\":\"atomic\"},\"gracePeriodSeconds\":{\"description\":\"The duration in seconds before the object should be deleted. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period for the specified type will be used. Defaults to a per object value if not specified. zero means delete immediately.\",\"format\":\"int64\",\"type\":\"integer\"},\"orphanDependents\":{\"description\":\"Deprecated: please use the PropagationPolicy, this field will be deprecated in 1.7. Should the dependent objects be orphaned. If true/false, the \\\"orphan\\\" finalizer will be added to/removed from the object's finalizers list. Either this field or PropagationPolicy may be set, but not both.\",\"type\":\"boolean\"},\"preconditions\":{\"allOf\":[{\"description\":\"Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.\",\"properties\":{\"resourceVersion\":{\"description\":\"Specifies the target ResourceVersion\",\"type\":\"string\"},\"uid\":{\"description\":\"Specifies the target UID.\",\"type\":\"string\"}},\"type\":\"object\"}],\"description\":\"Must be fulfilled before a deletion is carried out. If not possible, a 409 Conflict status will be returned.\"},\"propagationPolicy\":{\"description\":\"Whether and how garbage collection will be performed. Either this field or OrphanDependents may be set, but not both. The default policy is decided by the existing finalizer set in the metadata.finalizers and the resource-specific default policy. Acceptable values are: 'Orphan' - orphan the dependents; 'Background' - allow the garbage collector to delete the dependents in the background; 'Foreground' - a cascading policy that deletes all dependents in the foreground.\",\"type\":\"string\"}},\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"admission.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"admission.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"admissionregistration.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"admissionregistration.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"admissionregistration.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"apiextensions.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"apiextensions.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"apiregistration.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"apiregistration.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"apps\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"apps\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"apps\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta2\"},{\"group\":\"authentication.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"authentication.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"authentication.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"authorization.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"authorization.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"autoscaling\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"autoscaling\",\"kind\":\"DeleteOptions\",\"version\":\"v2\"},{\"group\":\"autoscaling\",\"kind\":\"DeleteOptions\",\"version\":\"v2beta1\"},{\"group\":\"autoscaling\",\"kind\":\"DeleteOptions\",\"version\":\"v2beta2\"},{\"group\":\"batch\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"batch\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"certificates.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"certificates.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"certificates.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"coordination.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"coordination.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"coordination.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"discovery.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"discovery.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"events.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"events.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"extensions\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta2\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta3\"},{\"group\":\"imagepolicy.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"internal.apiserver.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"networking.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"networking.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"networking.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"node.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"node.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"node.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"policy\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"policy\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"rbac.authorization.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"rbac.authorization.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"rbac.authorization.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"resource.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha3\"},{\"group\":\"scheduling.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"scheduling.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"scheduling.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"storage.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"storage.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"storage.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"storagemigration.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"}]}", +"version": "v1" +}, +"configuration": null, +"description": "", +"displayName": "deleteoptions", +"format": "JSON", +"id": "00000000-0000-0000-0000-000000000000", +"metadata": { +"genealogy": "", +"isAnnotation": false, +"isNamespaced": false, +"published": false +}, +"model": { +"category": { +"name": "Orchestration \u0026 Management" +}, +"displayName": "Kubernetes", +"id": "00000000-0000-0000-0000-000000000000", +"metadata": { +"source_uri": "https://raw.githubusercontent.com/kubernetes/kubernetes/refs/heads/master/api/openapi-spec/v3/apis__autoscaling__v1_openapi.json/v1.31.3", +"svgColor": "", +"svgWhite": "" +}, +"model": { +"version": "version" +}, +"name": "kubernetes", +"registrant": { +"created_at": "0001-01-01T00:00:00Z", +"credential_id": "00000000-0000-0000-0000-000000000000", +"deleted_at": "0001-01-01T00:00:00Z", +"id": "00000000-0000-0000-0000-000000000000", +"kind": "", +"name": "", +"status": "", +"sub_type": "", +"type": "", +"updated_at": "0001-01-01T00:00:00Z", +"user_id": "00000000-0000-0000-0000-000000000000" +}, +"connection_id": "00000000-0000-0000-0000-000000000000", +"schemaVersion": "models.meshery.io/v1beta1", +"status": "", +"version": "v1.0.0", +"components": null, +"relationships": null +}, +"schemaVersion": "components.meshery.io/v1beta1", +"status": null, +"styles": null, +"version": "v1.0.0" +} \ No newline at end of file diff --git a/generators/github/kubernetes/horizontalpodautoscaler.json b/generators/github/kubernetes/horizontalpodautoscaler.json new file mode 100644 index 00000000..764e2e1d --- /dev/null +++ b/generators/github/kubernetes/horizontalpodautoscaler.json @@ -0,0 +1,57 @@ +{ +"component": { +"kind": "horizontalpodautoscaler", +"schema": "{\"description\":\"configuration of a horizontal pod autoscaler.\",\"properties\":{\"spec\":{\"allOf\":[{\"description\":\"specification of a horizontal pod autoscaler.\",\"properties\":{\"maxReplicas\":{\"default\":0,\"description\":\"maxReplicas is the upper limit for the number of pods that can be set by the autoscaler; cannot be smaller than MinReplicas.\",\"format\":\"int32\",\"type\":\"integer\"},\"minReplicas\":{\"description\":\"minReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. It defaults to 1 pod. minReplicas is allowed to be 0 if the alpha feature gate HPAScaleToZero is enabled and at least one Object or External metric is configured. Scaling is active as long as at least one metric value is available.\",\"format\":\"int32\",\"type\":\"integer\"},\"scaleTargetRef\":{\"allOf\":[{\"description\":\"CrossVersionObjectReference contains enough information to let you identify the referred resource.\",\"properties\":{\"apiVersion\":{\"description\":\"apiVersion is the API version of the referent\",\"type\":\"string\"},\"kind\":{\"default\":\"\",\"description\":\"kind is the kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"default\":\"\",\"description\":\"name is the name of the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"}},\"required\":[\"kind\",\"name\"],\"type\":\"object\",\"x-kubernetes-map-type\":\"atomic\"}],\"default\":{},\"description\":\"reference to scaled resource; horizontal pod autoscaler will learn the current resource consumption and will set the desired number of pods by using its Scale subresource.\"},\"targetCPUUtilizationPercentage\":{\"description\":\"targetCPUUtilizationPercentage is the target average CPU utilization (represented as a percentage of requested CPU) over all the pods; if not specified the default autoscaling policy will be used.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"scaleTargetRef\",\"maxReplicas\"],\"type\":\"object\"}],\"default\":{},\"description\":\"spec defines the behaviour of autoscaler. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status.\"}},\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"autoscaling\",\"kind\":\"HorizontalPodAutoscaler\",\"version\":\"v1\"}]}", +"version": "autoscaling/v1" +}, +"configuration": null, +"description": "", +"displayName": "horizontalpodautoscaler", +"format": "JSON", +"id": "00000000-0000-0000-0000-000000000000", +"metadata": { +"genealogy": "", +"isAnnotation": false, +"isNamespaced": false, +"published": false +}, +"model": { +"category": { +"name": "Orchestration \u0026 Management" +}, +"displayName": "Kubernetes", +"id": "00000000-0000-0000-0000-000000000000", +"metadata": { +"source_uri": "https://raw.githubusercontent.com/kubernetes/kubernetes/refs/heads/master/api/openapi-spec/v3/apis__autoscaling__v1_openapi.json/v1.31.3", +"svgColor": "", +"svgWhite": "" +}, +"model": { +"version": "version" +}, +"name": "kubernetes", +"registrant": { +"created_at": "0001-01-01T00:00:00Z", +"credential_id": "00000000-0000-0000-0000-000000000000", +"deleted_at": "0001-01-01T00:00:00Z", +"id": "00000000-0000-0000-0000-000000000000", +"kind": "", +"name": "", +"status": "", +"sub_type": "", +"type": "", +"updated_at": "0001-01-01T00:00:00Z", +"user_id": "00000000-0000-0000-0000-000000000000" +}, +"connection_id": "00000000-0000-0000-0000-000000000000", +"schemaVersion": "models.meshery.io/v1beta1", +"status": "", +"version": "v1.0.0", +"components": null, +"relationships": null +}, +"schemaVersion": "components.meshery.io/v1beta1", +"status": null, +"styles": null, +"version": "v1.0.0" +} \ No newline at end of file diff --git a/generators/github/kubernetes/horizontalpodautoscalerlist.json b/generators/github/kubernetes/horizontalpodautoscalerlist.json new file mode 100644 index 00000000..175ce10b --- /dev/null +++ b/generators/github/kubernetes/horizontalpodautoscalerlist.json @@ -0,0 +1,57 @@ +{ +"component": { +"kind": "horizontalpodautoscalerlist", +"schema": "{\"description\":\"list of horizontal pod autoscaler objects.\",\"properties\":{\"items\":{\"description\":\"items is the list of horizontal pod autoscaler objects.\",\"items\":{\"allOf\":[{\"description\":\"configuration of a horizontal pod autoscaler.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"allOf\":[{\"description\":\"ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.\",\"properties\":{\"annotations\":{\"additionalProperties\":{\"default\":\"\",\"type\":\"string\"},\"description\":\"Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations\",\"type\":\"object\"},\"creationTimestamp\":{\"allOf\":[{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"}],\"description\":\"CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.\\n\\nPopulated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata\"},\"deletionGracePeriodSeconds\":{\"description\":\"Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"deletionTimestamp\":{\"allOf\":[{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"}],\"description\":\"DeletionTimestamp is RFC 3339 date and time at which this resource will be deleted. This field is set by the server when a graceful deletion is requested by the user, and is not directly settable by a client. The resource is expected to be deleted (no longer visible from resource lists, and not reachable by name) after the time in this field, once the finalizers list is empty. As long as the finalizers list contains items, deletion is blocked. Once the deletionTimestamp is set, this value may not be unset or be set further into the future, although it may be shortened or the resource may be deleted prior to this time. For example, a user may request that a pod is deleted in 30 seconds. The Kubelet will react by sending a graceful termination signal to the containers in the pod. After that 30 seconds, the Kubelet will send a hard termination signal (SIGKILL) to the container and after cleanup, remove the pod from the API. In the presence of network partitions, this object may still exist after this timestamp, until an administrator or automated process can determine the resource is fully terminated. If not set, graceful deletion of the object has not been requested.\\n\\nPopulated by the system when a graceful deletion is requested. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata\"},\"finalizers\":{\"description\":\"Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed. Finalizers may be processed and removed in any order. Order is NOT enforced because it introduces significant risk of stuck finalizers. finalizers is a shared field, any actor with permission can reorder it. If the finalizer list is processed in order, then this can lead to a situation in which the component responsible for the first finalizer in the list is waiting for a signal (field value, external system, or other) produced by a component responsible for a finalizer later in the list, resulting in a deadlock. Without enforced ordering finalizers are free to order amongst themselves and are not vulnerable to ordering changes in the list.\",\"items\":{\"default\":\"\",\"type\":\"string\"},\"type\":\"array\",\"x-kubernetes-list-type\":\"set\",\"x-kubernetes-patch-strategy\":\"merge\"},\"generateName\":{\"description\":\"GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\\n\\nIf this field is specified and the generated name exists, the server will return a 409.\\n\\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency\",\"type\":\"string\"},\"generation\":{\"description\":\"A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"labels\":{\"additionalProperties\":{\"default\":\"\",\"type\":\"string\"},\"description\":\"Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels\",\"type\":\"object\"},\"managedFields\":{\"description\":\"ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \\\"ci-cd\\\". The set of fields is always in the version that the workflow used when modifying the object.\",\"items\":{\"allOf\":[{\"description\":\"ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the version of this resource that this field set applies to. The format is \\\"group/version\\\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.\",\"type\":\"string\"},\"fieldsType\":{\"description\":\"FieldsType is the discriminator for the different fields format and version. There is currently only one possible value: \\\"FieldsV1\\\"\",\"type\":\"string\"},\"fieldsV1\":{\"allOf\":[{\"description\":\"FieldsV1 stores a set of fields in a data structure like a Trie, in JSON format.\\n\\nEach key is either a '.' representing the field itself, and will always map to an empty set, or a string representing a sub-field or item. The string will follow one of these four formats: 'f:\\u003cname\\u003e', where \\u003cname\\u003e is the name of a field in a struct, or key in a map 'v:\\u003cvalue\\u003e', where \\u003cvalue\\u003e is the exact json formatted value of a list item 'i:\\u003cindex\\u003e', where \\u003cindex\\u003e is position of a item in a list 'k:\\u003ckeys\\u003e', where \\u003ckeys\\u003e is a map of a list item's key fields to their unique values If a key maps to an empty Fields value, the field that key represents is part of the set.\\n\\nThe exact format is defined in sigs.k8s.io/structured-merge-diff\",\"type\":\"object\"}],\"description\":\"FieldsV1 holds the first JSON version format as described in the \\\"FieldsV1\\\" type.\"},\"manager\":{\"description\":\"Manager is an identifier of the workflow managing these fields.\",\"type\":\"string\"},\"operation\":{\"description\":\"Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.\",\"type\":\"string\"},\"subresource\":{\"description\":\"Subresource is the name of the subresource used to update that object, or empty string if the object was updated through the main resource. The value of this field is used to distinguish between managers, even if they share the same name. For example, a status update will be distinct from a regular update using the same manager name. Note that the APIVersion field is not related to the Subresource field and it always corresponds to the version of the main resource.\",\"type\":\"string\"},\"time\":{\"allOf\":[{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"}],\"description\":\"Time is the timestamp of when the ManagedFields entry was added. The timestamp will also be updated if a field is added, the manager changes any of the owned fields value or removes a field. The timestamp does not update when a field is removed from the entry because another manager took it over.\"}},\"type\":\"object\"}],\"default\":{}},\"type\":\"array\",\"x-kubernetes-list-type\":\"atomic\"},\"name\":{\"description\":\"Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace defines the space within which each name must be unique. An empty namespace is equivalent to the \\\"default\\\" namespace, but \\\"default\\\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\\n\\nMust be a DNS_LABEL. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces\",\"type\":\"string\"},\"ownerReferences\":{\"description\":\"List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.\",\"items\":{\"allOf\":[{\"description\":\"OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.\",\"properties\":{\"apiVersion\":{\"default\":\"\",\"description\":\"API version of the referent.\",\"type\":\"string\"},\"blockOwnerDeletion\":{\"description\":\"If true, AND if the owner has the \\\"foregroundDeletion\\\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. See https://kubernetes.io/docs/concepts/architecture/garbage-collection/#foreground-deletion for how the garbage collector interacts with this field and enforces the foreground deletion. Defaults to false. To set this field, a user needs \\\"delete\\\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.\",\"type\":\"boolean\"},\"controller\":{\"description\":\"If true, this reference points to the managing controller.\",\"type\":\"boolean\"},\"kind\":{\"default\":\"\",\"description\":\"Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"default\":\"\",\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names\",\"type\":\"string\"},\"uid\":{\"default\":\"\",\"description\":\"UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#uids\",\"type\":\"string\"}},\"required\":[\"apiVersion\",\"kind\",\"name\",\"uid\"],\"type\":\"object\",\"x-kubernetes-map-type\":\"atomic\"}],\"default\":{}},\"type\":\"array\",\"x-kubernetes-list-map-keys\":[\"uid\"],\"x-kubernetes-list-type\":\"map\",\"x-kubernetes-patch-merge-key\":\"uid\",\"x-kubernetes-patch-strategy\":\"merge\"},\"resourceVersion\":{\"description\":\"An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\\n\\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"Deprecated: selfLink is a legacy read-only field that is no longer populated by the system.\",\"type\":\"string\"},\"uid\":{\"description\":\"UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\\n\\nPopulated by the system. Read-only. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#uids\",\"type\":\"string\"}},\"type\":\"object\"}],\"default\":{},\"description\":\"Standard object metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata\"},\"spec\":{\"allOf\":[{\"description\":\"specification of a horizontal pod autoscaler.\",\"properties\":{\"maxReplicas\":{\"default\":0,\"description\":\"maxReplicas is the upper limit for the number of pods that can be set by the autoscaler; cannot be smaller than MinReplicas.\",\"format\":\"int32\",\"type\":\"integer\"},\"minReplicas\":{\"description\":\"minReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. It defaults to 1 pod. minReplicas is allowed to be 0 if the alpha feature gate HPAScaleToZero is enabled and at least one Object or External metric is configured. Scaling is active as long as at least one metric value is available.\",\"format\":\"int32\",\"type\":\"integer\"},\"scaleTargetRef\":{\"allOf\":[{\"description\":\"CrossVersionObjectReference contains enough information to let you identify the referred resource.\",\"properties\":{\"apiVersion\":{\"description\":\"apiVersion is the API version of the referent\",\"type\":\"string\"},\"kind\":{\"default\":\"\",\"description\":\"kind is the kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"default\":\"\",\"description\":\"name is the name of the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"}},\"required\":[\"kind\",\"name\"],\"type\":\"object\",\"x-kubernetes-map-type\":\"atomic\"}],\"default\":{},\"description\":\"reference to scaled resource; horizontal pod autoscaler will learn the current resource consumption and will set the desired number of pods by using its Scale subresource.\"},\"targetCPUUtilizationPercentage\":{\"description\":\"targetCPUUtilizationPercentage is the target average CPU utilization (represented as a percentage of requested CPU) over all the pods; if not specified the default autoscaling policy will be used.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"scaleTargetRef\",\"maxReplicas\"],\"type\":\"object\"}],\"default\":{},\"description\":\"spec defines the behaviour of autoscaler. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status.\"},\"status\":{\"allOf\":[{\"description\":\"current status of a horizontal pod autoscaler\",\"properties\":{\"currentCPUUtilizationPercentage\":{\"description\":\"currentCPUUtilizationPercentage is the current average CPU utilization over all pods, represented as a percentage of requested CPU, e.g. 70 means that an average pod is using now 70% of its requested CPU.\",\"format\":\"int32\",\"type\":\"integer\"},\"currentReplicas\":{\"default\":0,\"description\":\"currentReplicas is the current number of replicas of pods managed by this autoscaler.\",\"format\":\"int32\",\"type\":\"integer\"},\"desiredReplicas\":{\"default\":0,\"description\":\"desiredReplicas is the desired number of replicas of pods managed by this autoscaler.\",\"format\":\"int32\",\"type\":\"integer\"},\"lastScaleTime\":{\"allOf\":[{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"}],\"description\":\"lastScaleTime is the last time the HorizontalPodAutoscaler scaled the number of pods; used by the autoscaler to control how often the number of pods is changed.\"},\"observedGeneration\":{\"description\":\"observedGeneration is the most recent generation observed by this autoscaler.\",\"format\":\"int64\",\"type\":\"integer\"}},\"required\":[\"currentReplicas\",\"desiredReplicas\"],\"type\":\"object\"}],\"default\":{},\"description\":\"status is the current information about the autoscaler.\"}},\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"autoscaling\",\"kind\":\"HorizontalPodAutoscaler\",\"version\":\"v1\"}]}],\"default\":{}},\"type\":\"array\"}},\"required\":[\"items\"],\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"autoscaling\",\"kind\":\"HorizontalPodAutoscalerList\",\"version\":\"v1\"}]}", +"version": "autoscaling/v1" +}, +"configuration": null, +"description": "", +"displayName": "horizontalpodautoscalerlist", +"format": "JSON", +"id": "00000000-0000-0000-0000-000000000000", +"metadata": { +"genealogy": "", +"isAnnotation": false, +"isNamespaced": false, +"published": false +}, +"model": { +"category": { +"name": "Orchestration \u0026 Management" +}, +"displayName": "Kubernetes", +"id": "00000000-0000-0000-0000-000000000000", +"metadata": { +"source_uri": "https://raw.githubusercontent.com/kubernetes/kubernetes/refs/heads/master/api/openapi-spec/v3/apis__autoscaling__v1_openapi.json/v1.31.3", +"svgColor": "", +"svgWhite": "" +}, +"model": { +"version": "version" +}, +"name": "kubernetes", +"registrant": { +"created_at": "0001-01-01T00:00:00Z", +"credential_id": "00000000-0000-0000-0000-000000000000", +"deleted_at": "0001-01-01T00:00:00Z", +"id": "00000000-0000-0000-0000-000000000000", +"kind": "", +"name": "", +"status": "", +"sub_type": "", +"type": "", +"updated_at": "0001-01-01T00:00:00Z", +"user_id": "00000000-0000-0000-0000-000000000000" +}, +"connection_id": "00000000-0000-0000-0000-000000000000", +"schemaVersion": "models.meshery.io/v1beta1", +"status": "", +"version": "v1.0.0", +"components": null, +"relationships": null +}, +"schemaVersion": "components.meshery.io/v1beta1", +"status": null, +"styles": null, +"version": "v1.0.0" +} \ No newline at end of file diff --git a/generators/github/kubernetes/status.json b/generators/github/kubernetes/status.json new file mode 100644 index 00000000..2eef9691 --- /dev/null +++ b/generators/github/kubernetes/status.json @@ -0,0 +1,57 @@ +{ +"component": { +"kind": "status", +"schema": "{\"description\":\"Status is a return value for calls that don't return other objects.\",\"properties\":{\"code\":{\"description\":\"Suggested HTTP return code for this status, 0 if not set.\",\"format\":\"int32\",\"type\":\"integer\"},\"details\":{\"allOf\":[{\"description\":\"StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.\",\"properties\":{\"causes\":{\"description\":\"The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.\",\"items\":{\"allOf\":[{\"description\":\"StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.\",\"properties\":{\"field\":{\"description\":\"The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\\n\\nExamples:\\n \\\"name\\\" - the field \\\"name\\\" on the current resource\\n \\\"items[0].name\\\" - the field \\\"name\\\" on the first array entry in \\\"items\\\"\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the cause of the error. This field may be presented as-is to a reader.\",\"type\":\"string\"},\"reason\":{\"description\":\"A machine-readable description of the cause of the error. If this value is empty there is no information available.\",\"type\":\"string\"}},\"type\":\"object\"}],\"default\":{}},\"type\":\"array\",\"x-kubernetes-list-type\":\"atomic\"},\"group\":{\"description\":\"The group attribute of the resource associated with the status StatusReason.\",\"type\":\"string\"},\"kind\":{\"description\":\"The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).\",\"type\":\"string\"},\"retryAfterSeconds\":{\"description\":\"If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.\",\"format\":\"int32\",\"type\":\"integer\"},\"uid\":{\"description\":\"UID of the resource. (when there is a single resource which can be described). More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#uids\",\"type\":\"string\"}},\"type\":\"object\"}],\"description\":\"Extended data associated with the reason. Each reason may define its own extended details. This field is optional and the data returned is not guaranteed to conform to any schema except that defined by the reason type.\",\"x-kubernetes-list-type\":\"atomic\"},\"message\":{\"description\":\"A human-readable description of the status of this operation.\",\"type\":\"string\"},\"reason\":{\"description\":\"A machine-readable description of why this operation is in the \\\"Failure\\\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.\",\"type\":\"string\"}},\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"\",\"kind\":\"Status\",\"version\":\"v1\"}]}", +"version": "v1" +}, +"configuration": null, +"description": "", +"displayName": "status", +"format": "JSON", +"id": "00000000-0000-0000-0000-000000000000", +"metadata": { +"genealogy": "", +"isAnnotation": false, +"isNamespaced": false, +"published": false +}, +"model": { +"category": { +"name": "Orchestration \u0026 Management" +}, +"displayName": "Kubernetes", +"id": "00000000-0000-0000-0000-000000000000", +"metadata": { +"source_uri": "https://raw.githubusercontent.com/kubernetes/kubernetes/refs/heads/master/api/openapi-spec/v3/apis__autoscaling__v1_openapi.json/v1.31.3", +"svgColor": "", +"svgWhite": "" +}, +"model": { +"version": "version" +}, +"name": "kubernetes", +"registrant": { +"created_at": "0001-01-01T00:00:00Z", +"credential_id": "00000000-0000-0000-0000-000000000000", +"deleted_at": "0001-01-01T00:00:00Z", +"id": "00000000-0000-0000-0000-000000000000", +"kind": "", +"name": "", +"status": "", +"sub_type": "", +"type": "", +"updated_at": "0001-01-01T00:00:00Z", +"user_id": "00000000-0000-0000-0000-000000000000" +}, +"connection_id": "00000000-0000-0000-0000-000000000000", +"schemaVersion": "models.meshery.io/v1beta1", +"status": "", +"version": "v1.0.0", +"components": null, +"relationships": null +}, +"schemaVersion": "components.meshery.io/v1beta1", +"status": null, +"styles": null, +"version": "v1.0.0" +} \ No newline at end of file diff --git a/generators/github/kubernetes/watchevent.json b/generators/github/kubernetes/watchevent.json new file mode 100644 index 00000000..342c16f8 --- /dev/null +++ b/generators/github/kubernetes/watchevent.json @@ -0,0 +1,57 @@ +{ +"component": { +"kind": "watchevent", +"schema": "{\"description\":\"Event represents a single event to a watched resource.\",\"properties\":{\"object\":{\"allOf\":[{\"description\":\"RawExtension is used to hold extensions in external versions.\\n\\nTo use this, make a field which has RawExtension as its type in your external, versioned struct, and Object in your internal struct. You also need to register your various plugin types.\\n\\n// Internal package:\\n\\n\\ttype MyAPIObject struct {\\n\\t\\truntime.TypeMeta `json:\\\",inline\\\"`\\n\\t\\tMyPlugin runtime.Object `json:\\\"myPlugin\\\"`\\n\\t}\\n\\n\\ttype PluginA struct {\\n\\t\\tAOption string `json:\\\"aOption\\\"`\\n\\t}\\n\\n// External package:\\n\\n\\ttype MyAPIObject struct {\\n\\t\\truntime.TypeMeta `json:\\\",inline\\\"`\\n\\t\\tMyPlugin runtime.RawExtension `json:\\\"myPlugin\\\"`\\n\\t}\\n\\n\\ttype PluginA struct {\\n\\t\\tAOption string `json:\\\"aOption\\\"`\\n\\t}\\n\\n// On the wire, the JSON will look something like this:\\n\\n\\t{\\n\\t\\t\\\"kind\\\":\\\"MyAPIObject\\\",\\n\\t\\t\\\"apiVersion\\\":\\\"v1\\\",\\n\\t\\t\\\"myPlugin\\\": {\\n\\t\\t\\t\\\"kind\\\":\\\"PluginA\\\",\\n\\t\\t\\t\\\"aOption\\\":\\\"foo\\\",\\n\\t\\t},\\n\\t}\\n\\nSo what happens? Decode first uses json or yaml to unmarshal the serialized data into your external MyAPIObject. That causes the raw JSON to be stored, but not unpacked. The next step is to copy (using pkg/conversion) into the internal struct. The runtime package's DefaultScheme has conversion functions installed which will unpack the JSON stored in RawExtension, turning it into the correct object type, and storing it in the Object. (TODO: In the case where the object is of an unknown type, a runtime.Unknown object will be created and stored.)\",\"type\":\"object\"}],\"description\":\"Object is:\\n * If Type is Added or Modified: the new state of the object.\\n * If Type is Deleted: the state of the object immediately before deletion.\\n * If Type is Error: *Status is recommended; other types may make sense\\n depending on context.\"},\"type\":{\"default\":\"\",\"type\":\"string\"}},\"required\":[\"type\",\"object\"],\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"admission.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"admission.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"admissionregistration.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"admissionregistration.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"admissionregistration.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"apiextensions.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"apiextensions.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"apiregistration.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"apiregistration.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"apps\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"apps\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"apps\",\"kind\":\"WatchEvent\",\"version\":\"v1beta2\"},{\"group\":\"authentication.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"authentication.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"authentication.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"authorization.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"authorization.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"autoscaling\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"autoscaling\",\"kind\":\"WatchEvent\",\"version\":\"v2\"},{\"group\":\"autoscaling\",\"kind\":\"WatchEvent\",\"version\":\"v2beta1\"},{\"group\":\"autoscaling\",\"kind\":\"WatchEvent\",\"version\":\"v2beta2\"},{\"group\":\"batch\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"batch\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"certificates.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"certificates.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"certificates.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"coordination.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"coordination.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"coordination.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"discovery.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"discovery.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"events.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"events.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"extensions\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta2\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta3\"},{\"group\":\"imagepolicy.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"internal.apiserver.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"networking.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"networking.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"networking.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"node.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"node.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"node.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"policy\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"policy\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"rbac.authorization.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"rbac.authorization.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"rbac.authorization.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"resource.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha3\"},{\"group\":\"scheduling.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"scheduling.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"scheduling.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"storage.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"storage.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"storage.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"storagemigration.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"}]}", +"version": "v1" +}, +"configuration": null, +"description": "", +"displayName": "watchevent", +"format": "JSON", +"id": "00000000-0000-0000-0000-000000000000", +"metadata": { +"genealogy": "", +"isAnnotation": false, +"isNamespaced": false, +"published": false +}, +"model": { +"category": { +"name": "Orchestration \u0026 Management" +}, +"displayName": "Kubernetes", +"id": "00000000-0000-0000-0000-000000000000", +"metadata": { +"source_uri": "https://raw.githubusercontent.com/kubernetes/kubernetes/refs/heads/master/api/openapi-spec/v3/apis__autoscaling__v1_openapi.json/v1.31.3", +"svgColor": "", +"svgWhite": "" +}, +"model": { +"version": "version" +}, +"name": "kubernetes", +"registrant": { +"created_at": "0001-01-01T00:00:00Z", +"credential_id": "00000000-0000-0000-0000-000000000000", +"deleted_at": "0001-01-01T00:00:00Z", +"id": "00000000-0000-0000-0000-000000000000", +"kind": "", +"name": "", +"status": "", +"sub_type": "", +"type": "", +"updated_at": "0001-01-01T00:00:00Z", +"user_id": "00000000-0000-0000-0000-000000000000" +}, +"connection_id": "00000000-0000-0000-0000-000000000000", +"schemaVersion": "models.meshery.io/v1beta1", +"status": "", +"version": "v1.0.0", +"components": null, +"relationships": null +}, +"schemaVersion": "components.meshery.io/v1beta1", +"status": null, +"styles": null, +"version": "v1.0.0" +} \ No newline at end of file From e75742849f19b96f4344d27cdedfaf49197cd635 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 25 Sep 2024 14:07:59 +0530 Subject: [PATCH 090/112] fix import Signed-off-by: MUzairS15 --- utils/kubernetes/crd.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/utils/kubernetes/crd.go b/utils/kubernetes/crd.go index de813120..9bf8d991 100644 --- a/utils/kubernetes/crd.go +++ b/utils/kubernetes/crd.go @@ -4,6 +4,7 @@ import ( "context" "github.com/layer5io/meshkit/encoding" + "github.com/layer5io/meshkit/utils" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" ) @@ -54,18 +55,18 @@ func GetGVRForCustomResources(crd *CRDItem) *schema.GroupVersionResource { } func IsCRD(manifest string) bool { - cueValue, err := utils.YamlToCue(manifest) - if err!= nil { - return false - } - kind, err := utils.Lookup(cueValue, "kind") - if err!= nil { - return false - } - kindStr, err := kind.String() + cueValue, err := utils.YamlToCue(manifest) if err != nil { - return false - } + return false + } + kind, err := utils.Lookup(cueValue, "kind") + if err != nil { + return false + } + kindStr, err := kind.String() + if err != nil { + return false + } - return kindStr == "CustomResourceDefinition" -} \ No newline at end of file + return kindStr == "CustomResourceDefinition" +} From 112a538180174fea091f744dfae28f7ee29e8f69 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 25 Sep 2024 15:26:16 +0530 Subject: [PATCH 091/112] define new methods on package interface Signed-off-by: MUzairS15 --- generators/artifacthub/package.go | 8 ++++++++ generators/github/package.go | 10 +++++++++- generators/models/interfaces.go | 2 ++ utils/component/openapi_generator.go | 11 ++++++----- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index 61b3d8be..e40509d8 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -38,6 +38,14 @@ func (pkg AhPackage) GetVersion() string { return pkg.Version } +func (pkg AhPackage) GetSourceURL() string { + return pkg.ChartUrl +} + +func (pkg AhPackage) GetName() string { + return pkg.Name +} + func (pkg AhPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { components := make([]_component.ComponentDefinition, 0) // TODO: Move this to the configuration diff --git a/generators/github/package.go b/generators/github/package.go index 7f083bf3..aff3be4b 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -26,6 +26,14 @@ func (gp GitHubPackage) GetVersion() string { return gp.version } +func (gp GitHubPackage) GetSourceURL() string { + return gp.SourceURL +} + +func (gp GitHubPackage) GetName() string { + return gp.Name +} + func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { components := make([]_component.ComponentDefinition, 0) @@ -41,7 +49,7 @@ func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, isCrd := kubernetes.IsCRD(string(crd)) if !isCrd { - comps, err := component.GenerateFromOpenAPI(string(crd), gp.SourceURL) + comps, err := component.GenerateFromOpenAPI(string(crd), gp) if err != nil { errs = append(errs, err) continue diff --git a/generators/models/interfaces.go b/generators/models/interfaces.go index 19f61177..c6dcd3eb 100644 --- a/generators/models/interfaces.go +++ b/generators/models/interfaces.go @@ -13,6 +13,8 @@ type Validator interface { type Package interface { GenerateComponents() ([]component.ComponentDefinition, error) GetVersion() string + GetSourceURL() string + GetName() string } // Supports pulling packages from Artifact Hub and other sources like Docker Hub. diff --git a/utils/component/openapi_generator.go b/utils/component/openapi_generator.go index 25c28de4..12d37a59 100644 --- a/utils/component/openapi_generator.go +++ b/utils/component/openapi_generator.go @@ -8,6 +8,7 @@ import ( "cuelang.org/go/cue" "cuelang.org/go/cue/cuecontext" cueJson "cuelang.org/go/encoding/json" + "github.com/layer5io/meshkit/generators/models" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/manifests" @@ -19,7 +20,7 @@ import ( "github.com/meshery/schemas/models/v1beta1/model" ) -func GenerateFromOpenAPI(resource, sourceURL string) ([]component.ComponentDefinition, error) { +func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.ComponentDefinition, error) { if resource == "" { return nil, nil } @@ -103,16 +104,16 @@ func GenerateFromOpenAPI(resource, sourceURL string) ([]component.ComponentDefin Version: "v1.0.0", Model: model.Model{ - Version: "version", + Version: pkg.GetVersion(), }, - Name: "kubernetes", - DisplayName: "Kubernetes", + Name: pkg.GetName(), + DisplayName: manifests.FormatToReadableString(pkg.GetName()), Category: category.CategoryDefinition{ Name: "Orchestration & Management", }, Metadata: &model.ModelDefinition_Metadata{ AdditionalProperties: map[string]interface{}{ - "source_uri": sourceURL, + "source_uri": pkg.GetSourceURL(), }, }, }, From 3246b89d6a3a92fb2dac561d3ef4830786dc0e48 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Fri, 27 Sep 2024 15:41:52 +0530 Subject: [PATCH 092/112] add sync for relationship w/signoff Signed-off-by: Jougan-0 --- models/meshmodel/core/policies/rego_policy_relationship.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index 8b2c60e0..f426dfd5 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -3,6 +3,7 @@ package policies import ( "context" "fmt" + "sync" "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/layer5io/meshkit/models/meshmodel/registry/v1alpha3" @@ -15,6 +16,8 @@ import ( "github.com/sirupsen/logrus" ) +var SyncRelationship sync.Mutex + type Rego struct { store storage.Store ctx context.Context From 08164fd911b3a9387a325f01f065ddcfcefda43e Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Fri, 27 Sep 2024 19:27:30 +0530 Subject: [PATCH 093/112] fix error handling Signed-off-by: MUzairS15 --- utils/kubernetes/client.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils/kubernetes/client.go b/utils/kubernetes/client.go index 6570fefe..9e554779 100644 --- a/utils/kubernetes/client.go +++ b/utils/kubernetes/client.go @@ -21,13 +21,13 @@ func DetectKubeConfig(configfile []byte) (config *rest.Config, err error) { } if config, err = clientcmd.RESTConfigFromKubeConfig(cfgFile); err == nil { - return config, ErrRestConfigFromKubeConfig(err) + return config, nil } } // If deployed within the cluster if config, err = rest.InClusterConfig(); err == nil { - return config, ErrRestConfigFromKubeConfig(err) + return config, nil } // Look for kubeconfig from the path mentioned in $KUBECONFIG @@ -38,7 +38,7 @@ func DetectKubeConfig(configfile []byte) (config *rest.Config, err error) { return nil, err } if config, err = clientcmd.RESTConfigFromKubeConfig(cfgFile); err == nil { - return config, ErrRestConfigFromKubeConfig(err) + return config, nil } } @@ -49,10 +49,10 @@ func DetectKubeConfig(configfile []byte) (config *rest.Config, err error) { return nil, err } if config, err = clientcmd.RESTConfigFromKubeConfig(cfgFile); err == nil { - return config, ErrRestConfigFromKubeConfig(err) + return config, nil } - return + return nil, ErrRestConfigFromKubeConfig(err) } // ProcessConfig handles loading, validating, and optionally saving or returning a kubeconfig From 12ec8f66b1b1a3b47628a56b181112b0b648d5d1 Mon Sep 17 00:00:00 2001 From: aabidsofi19 Date: Sun, 29 Sep 2024 22:23:41 +0530 Subject: [PATCH 094/112] Update import design schema Signed-off-by: aabidsofi19 --- schemas/configuration/designImport.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/schemas/configuration/designImport.json b/schemas/configuration/designImport.json index 4c5752af..f59b205b 100644 --- a/schemas/configuration/designImport.json +++ b/schemas/configuration/designImport.json @@ -3,7 +3,7 @@ "properties": { "name": { "type": "string", - "title": "Design file name", + "title": "Design filename", "default": "Untitled Design", "x-rjsf-grid-area": "6", "description": "Provide a name for your design file. This name will help you identify the file more easily. You can also change the name of your design after importing it." @@ -16,7 +16,6 @@ "Docker Compose", "Meshery Design" ], - "default": "Meshery Design", "x-rjsf-grid-area": "6", "description": "Select the type of design you are uploading. The 'Design Type' determines the format, structure, and content of the file you are uploading. Choose the appropriate design type that matches the nature of your file. Checkout https://docs.meshery.io/guides/meshery-design to learn more about designs" } @@ -126,4 +125,4 @@ ] } ] -} \ No newline at end of file +} From ad5c26539f105a8fff3dc2ef78bdd5ea08c6a2af Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 30 Sep 2024 15:31:27 +0530 Subject: [PATCH 095/112] remove unneeded files Signed-off-by: MUzairS15 --- .../github/kubernetes/apiresourcelist.json | 57 ------------------- .../github/kubernetes/deleteoptions.json | 57 ------------------- .../kubernetes/horizontalpodautoscaler.json | 57 ------------------- .../horizontalpodautoscalerlist.json | 57 ------------------- generators/github/kubernetes/status.json | 57 ------------------- generators/github/kubernetes/watchevent.json | 57 ------------------- 6 files changed, 342 deletions(-) delete mode 100644 generators/github/kubernetes/apiresourcelist.json delete mode 100644 generators/github/kubernetes/deleteoptions.json delete mode 100644 generators/github/kubernetes/horizontalpodautoscaler.json delete mode 100644 generators/github/kubernetes/horizontalpodautoscalerlist.json delete mode 100644 generators/github/kubernetes/status.json delete mode 100644 generators/github/kubernetes/watchevent.json diff --git a/generators/github/kubernetes/apiresourcelist.json b/generators/github/kubernetes/apiresourcelist.json deleted file mode 100644 index 8f9d8b74..00000000 --- a/generators/github/kubernetes/apiresourcelist.json +++ /dev/null @@ -1,57 +0,0 @@ -{ -"component": { -"kind": "apiresourcelist", -"schema": "{\"description\":\"APIResourceList is a list of APIResource, it is used to expose the name of the resources supported in a specific group and version, and if the resource is namespaced.\",\"properties\":{\"groupVersion\":{\"default\":\"\",\"description\":\"groupVersion is the group and version this APIResourceList is for.\",\"type\":\"string\"},\"resources\":{\"description\":\"resources contains the name of the resources and if they are namespaced.\",\"items\":{\"allOf\":[{\"description\":\"APIResource specifies the name of a resource and whether it is namespaced.\",\"properties\":{\"categories\":{\"description\":\"categories is a list of the grouped resources this resource belongs to (e.g. 'all')\",\"items\":{\"default\":\"\",\"type\":\"string\"},\"type\":\"array\",\"x-kubernetes-list-type\":\"atomic\"},\"group\":{\"description\":\"group is the preferred group of the resource. Empty implies the group of the containing resource list. For subresources, this may have a different value, for example: Scale\\\".\",\"type\":\"string\"},\"kind\":{\"default\":\"\",\"description\":\"kind is the kind for the resource (e.g. 'Foo' is the kind for a resource 'foo')\",\"type\":\"string\"},\"name\":{\"default\":\"\",\"description\":\"name is the plural name of the resource.\",\"type\":\"string\"},\"namespaced\":{\"default\":false,\"description\":\"namespaced indicates if a resource is namespaced or not.\",\"type\":\"boolean\"},\"shortNames\":{\"description\":\"shortNames is a list of suggested short names of the resource.\",\"items\":{\"default\":\"\",\"type\":\"string\"},\"type\":\"array\",\"x-kubernetes-list-type\":\"atomic\"},\"singularName\":{\"default\":\"\",\"description\":\"singularName is the singular name of the resource. This allows clients to handle plural and singular opaquely. The singularName is more correct for reporting status on a single item and both singular and plural are allowed from the kubectl CLI interface.\",\"type\":\"string\"},\"storageVersionHash\":{\"description\":\"The hash value of the storage version, the version this resource is converted to when written to the data store. Value must be treated as opaque by clients. Only equality comparison on the value is valid. This is an alpha feature and may change or be removed in the future. The field is populated by the apiserver only if the StorageVersionHash feature gate is enabled. This field will remain optional even if it graduates.\",\"type\":\"string\"},\"verbs\":{\"description\":\"verbs is a list of supported kube verbs (this includes get, list, watch, create, update, patch, delete, deletecollection, and proxy)\",\"items\":{\"default\":\"\",\"type\":\"string\"},\"type\":\"array\"},\"version\":{\"description\":\"version is the preferred version of the resource. Empty implies the version of the containing resource list For subresources, this may have a different value, for example: v1 (while inside a v1beta1 version of the core resource's group)\\\".\",\"type\":\"string\"}},\"required\":[\"name\",\"singularName\",\"namespaced\",\"kind\",\"verbs\"],\"type\":\"object\"}],\"default\":{}},\"type\":\"array\",\"x-kubernetes-list-type\":\"atomic\"}},\"required\":[\"groupVersion\",\"resources\"],\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"\",\"kind\":\"APIResourceList\",\"version\":\"v1\"}]}", -"version": "v1" -}, -"configuration": null, -"description": "", -"displayName": "apiresourcelist", -"format": "JSON", -"id": "00000000-0000-0000-0000-000000000000", -"metadata": { -"genealogy": "", -"isAnnotation": false, -"isNamespaced": false, -"published": false -}, -"model": { -"category": { -"name": "Orchestration \u0026 Management" -}, -"displayName": "Kubernetes", -"id": "00000000-0000-0000-0000-000000000000", -"metadata": { -"source_uri": "https://raw.githubusercontent.com/kubernetes/kubernetes/refs/heads/master/api/openapi-spec/v3/apis__autoscaling__v1_openapi.json/v1.31.3", -"svgColor": "", -"svgWhite": "" -}, -"model": { -"version": "version" -}, -"name": "kubernetes", -"registrant": { -"created_at": "0001-01-01T00:00:00Z", -"credential_id": "00000000-0000-0000-0000-000000000000", -"deleted_at": "0001-01-01T00:00:00Z", -"id": "00000000-0000-0000-0000-000000000000", -"kind": "", -"name": "", -"status": "", -"sub_type": "", -"type": "", -"updated_at": "0001-01-01T00:00:00Z", -"user_id": "00000000-0000-0000-0000-000000000000" -}, -"connection_id": "00000000-0000-0000-0000-000000000000", -"schemaVersion": "models.meshery.io/v1beta1", -"status": "", -"version": "v1.0.0", -"components": null, -"relationships": null -}, -"schemaVersion": "components.meshery.io/v1beta1", -"status": null, -"styles": null, -"version": "v1.0.0" -} \ No newline at end of file diff --git a/generators/github/kubernetes/deleteoptions.json b/generators/github/kubernetes/deleteoptions.json deleted file mode 100644 index 41ebe8d6..00000000 --- a/generators/github/kubernetes/deleteoptions.json +++ /dev/null @@ -1,57 +0,0 @@ -{ -"component": { -"kind": "deleteoptions", -"schema": "{\"description\":\"DeleteOptions may be provided when deleting an API object.\",\"properties\":{\"dryRun\":{\"description\":\"When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed\",\"items\":{\"default\":\"\",\"type\":\"string\"},\"type\":\"array\",\"x-kubernetes-list-type\":\"atomic\"},\"gracePeriodSeconds\":{\"description\":\"The duration in seconds before the object should be deleted. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period for the specified type will be used. Defaults to a per object value if not specified. zero means delete immediately.\",\"format\":\"int64\",\"type\":\"integer\"},\"orphanDependents\":{\"description\":\"Deprecated: please use the PropagationPolicy, this field will be deprecated in 1.7. Should the dependent objects be orphaned. If true/false, the \\\"orphan\\\" finalizer will be added to/removed from the object's finalizers list. Either this field or PropagationPolicy may be set, but not both.\",\"type\":\"boolean\"},\"preconditions\":{\"allOf\":[{\"description\":\"Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.\",\"properties\":{\"resourceVersion\":{\"description\":\"Specifies the target ResourceVersion\",\"type\":\"string\"},\"uid\":{\"description\":\"Specifies the target UID.\",\"type\":\"string\"}},\"type\":\"object\"}],\"description\":\"Must be fulfilled before a deletion is carried out. If not possible, a 409 Conflict status will be returned.\"},\"propagationPolicy\":{\"description\":\"Whether and how garbage collection will be performed. Either this field or OrphanDependents may be set, but not both. The default policy is decided by the existing finalizer set in the metadata.finalizers and the resource-specific default policy. Acceptable values are: 'Orphan' - orphan the dependents; 'Background' - allow the garbage collector to delete the dependents in the background; 'Foreground' - a cascading policy that deletes all dependents in the foreground.\",\"type\":\"string\"}},\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"admission.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"admission.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"admissionregistration.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"admissionregistration.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"admissionregistration.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"apiextensions.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"apiextensions.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"apiregistration.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"apiregistration.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"apps\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"apps\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"apps\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta2\"},{\"group\":\"authentication.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"authentication.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"authentication.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"authorization.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"authorization.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"autoscaling\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"autoscaling\",\"kind\":\"DeleteOptions\",\"version\":\"v2\"},{\"group\":\"autoscaling\",\"kind\":\"DeleteOptions\",\"version\":\"v2beta1\"},{\"group\":\"autoscaling\",\"kind\":\"DeleteOptions\",\"version\":\"v2beta2\"},{\"group\":\"batch\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"batch\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"certificates.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"certificates.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"certificates.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"coordination.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"coordination.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"coordination.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"discovery.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"discovery.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"events.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"events.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"extensions\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta2\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta3\"},{\"group\":\"imagepolicy.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"internal.apiserver.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"networking.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"networking.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"networking.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"node.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"node.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"node.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"policy\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"policy\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"rbac.authorization.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"rbac.authorization.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"rbac.authorization.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"resource.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha3\"},{\"group\":\"scheduling.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"scheduling.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"scheduling.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"storage.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1\"},{\"group\":\"storage.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"},{\"group\":\"storage.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1beta1\"},{\"group\":\"storagemigration.k8s.io\",\"kind\":\"DeleteOptions\",\"version\":\"v1alpha1\"}]}", -"version": "v1" -}, -"configuration": null, -"description": "", -"displayName": "deleteoptions", -"format": "JSON", -"id": "00000000-0000-0000-0000-000000000000", -"metadata": { -"genealogy": "", -"isAnnotation": false, -"isNamespaced": false, -"published": false -}, -"model": { -"category": { -"name": "Orchestration \u0026 Management" -}, -"displayName": "Kubernetes", -"id": "00000000-0000-0000-0000-000000000000", -"metadata": { -"source_uri": "https://raw.githubusercontent.com/kubernetes/kubernetes/refs/heads/master/api/openapi-spec/v3/apis__autoscaling__v1_openapi.json/v1.31.3", -"svgColor": "", -"svgWhite": "" -}, -"model": { -"version": "version" -}, -"name": "kubernetes", -"registrant": { -"created_at": "0001-01-01T00:00:00Z", -"credential_id": "00000000-0000-0000-0000-000000000000", -"deleted_at": "0001-01-01T00:00:00Z", -"id": "00000000-0000-0000-0000-000000000000", -"kind": "", -"name": "", -"status": "", -"sub_type": "", -"type": "", -"updated_at": "0001-01-01T00:00:00Z", -"user_id": "00000000-0000-0000-0000-000000000000" -}, -"connection_id": "00000000-0000-0000-0000-000000000000", -"schemaVersion": "models.meshery.io/v1beta1", -"status": "", -"version": "v1.0.0", -"components": null, -"relationships": null -}, -"schemaVersion": "components.meshery.io/v1beta1", -"status": null, -"styles": null, -"version": "v1.0.0" -} \ No newline at end of file diff --git a/generators/github/kubernetes/horizontalpodautoscaler.json b/generators/github/kubernetes/horizontalpodautoscaler.json deleted file mode 100644 index 764e2e1d..00000000 --- a/generators/github/kubernetes/horizontalpodautoscaler.json +++ /dev/null @@ -1,57 +0,0 @@ -{ -"component": { -"kind": "horizontalpodautoscaler", -"schema": "{\"description\":\"configuration of a horizontal pod autoscaler.\",\"properties\":{\"spec\":{\"allOf\":[{\"description\":\"specification of a horizontal pod autoscaler.\",\"properties\":{\"maxReplicas\":{\"default\":0,\"description\":\"maxReplicas is the upper limit for the number of pods that can be set by the autoscaler; cannot be smaller than MinReplicas.\",\"format\":\"int32\",\"type\":\"integer\"},\"minReplicas\":{\"description\":\"minReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. It defaults to 1 pod. minReplicas is allowed to be 0 if the alpha feature gate HPAScaleToZero is enabled and at least one Object or External metric is configured. Scaling is active as long as at least one metric value is available.\",\"format\":\"int32\",\"type\":\"integer\"},\"scaleTargetRef\":{\"allOf\":[{\"description\":\"CrossVersionObjectReference contains enough information to let you identify the referred resource.\",\"properties\":{\"apiVersion\":{\"description\":\"apiVersion is the API version of the referent\",\"type\":\"string\"},\"kind\":{\"default\":\"\",\"description\":\"kind is the kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"default\":\"\",\"description\":\"name is the name of the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"}},\"required\":[\"kind\",\"name\"],\"type\":\"object\",\"x-kubernetes-map-type\":\"atomic\"}],\"default\":{},\"description\":\"reference to scaled resource; horizontal pod autoscaler will learn the current resource consumption and will set the desired number of pods by using its Scale subresource.\"},\"targetCPUUtilizationPercentage\":{\"description\":\"targetCPUUtilizationPercentage is the target average CPU utilization (represented as a percentage of requested CPU) over all the pods; if not specified the default autoscaling policy will be used.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"scaleTargetRef\",\"maxReplicas\"],\"type\":\"object\"}],\"default\":{},\"description\":\"spec defines the behaviour of autoscaler. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status.\"}},\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"autoscaling\",\"kind\":\"HorizontalPodAutoscaler\",\"version\":\"v1\"}]}", -"version": "autoscaling/v1" -}, -"configuration": null, -"description": "", -"displayName": "horizontalpodautoscaler", -"format": "JSON", -"id": "00000000-0000-0000-0000-000000000000", -"metadata": { -"genealogy": "", -"isAnnotation": false, -"isNamespaced": false, -"published": false -}, -"model": { -"category": { -"name": "Orchestration \u0026 Management" -}, -"displayName": "Kubernetes", -"id": "00000000-0000-0000-0000-000000000000", -"metadata": { -"source_uri": "https://raw.githubusercontent.com/kubernetes/kubernetes/refs/heads/master/api/openapi-spec/v3/apis__autoscaling__v1_openapi.json/v1.31.3", -"svgColor": "", -"svgWhite": "" -}, -"model": { -"version": "version" -}, -"name": "kubernetes", -"registrant": { -"created_at": "0001-01-01T00:00:00Z", -"credential_id": "00000000-0000-0000-0000-000000000000", -"deleted_at": "0001-01-01T00:00:00Z", -"id": "00000000-0000-0000-0000-000000000000", -"kind": "", -"name": "", -"status": "", -"sub_type": "", -"type": "", -"updated_at": "0001-01-01T00:00:00Z", -"user_id": "00000000-0000-0000-0000-000000000000" -}, -"connection_id": "00000000-0000-0000-0000-000000000000", -"schemaVersion": "models.meshery.io/v1beta1", -"status": "", -"version": "v1.0.0", -"components": null, -"relationships": null -}, -"schemaVersion": "components.meshery.io/v1beta1", -"status": null, -"styles": null, -"version": "v1.0.0" -} \ No newline at end of file diff --git a/generators/github/kubernetes/horizontalpodautoscalerlist.json b/generators/github/kubernetes/horizontalpodautoscalerlist.json deleted file mode 100644 index 175ce10b..00000000 --- a/generators/github/kubernetes/horizontalpodautoscalerlist.json +++ /dev/null @@ -1,57 +0,0 @@ -{ -"component": { -"kind": "horizontalpodautoscalerlist", -"schema": "{\"description\":\"list of horizontal pod autoscaler objects.\",\"properties\":{\"items\":{\"description\":\"items is the list of horizontal pod autoscaler objects.\",\"items\":{\"allOf\":[{\"description\":\"configuration of a horizontal pod autoscaler.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\",\"type\":\"string\"},\"kind\":{\"description\":\"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"metadata\":{\"allOf\":[{\"description\":\"ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.\",\"properties\":{\"annotations\":{\"additionalProperties\":{\"default\":\"\",\"type\":\"string\"},\"description\":\"Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations\",\"type\":\"object\"},\"creationTimestamp\":{\"allOf\":[{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"}],\"description\":\"CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.\\n\\nPopulated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata\"},\"deletionGracePeriodSeconds\":{\"description\":\"Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"deletionTimestamp\":{\"allOf\":[{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"}],\"description\":\"DeletionTimestamp is RFC 3339 date and time at which this resource will be deleted. This field is set by the server when a graceful deletion is requested by the user, and is not directly settable by a client. The resource is expected to be deleted (no longer visible from resource lists, and not reachable by name) after the time in this field, once the finalizers list is empty. As long as the finalizers list contains items, deletion is blocked. Once the deletionTimestamp is set, this value may not be unset or be set further into the future, although it may be shortened or the resource may be deleted prior to this time. For example, a user may request that a pod is deleted in 30 seconds. The Kubelet will react by sending a graceful termination signal to the containers in the pod. After that 30 seconds, the Kubelet will send a hard termination signal (SIGKILL) to the container and after cleanup, remove the pod from the API. In the presence of network partitions, this object may still exist after this timestamp, until an administrator or automated process can determine the resource is fully terminated. If not set, graceful deletion of the object has not been requested.\\n\\nPopulated by the system when a graceful deletion is requested. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata\"},\"finalizers\":{\"description\":\"Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed. Finalizers may be processed and removed in any order. Order is NOT enforced because it introduces significant risk of stuck finalizers. finalizers is a shared field, any actor with permission can reorder it. If the finalizer list is processed in order, then this can lead to a situation in which the component responsible for the first finalizer in the list is waiting for a signal (field value, external system, or other) produced by a component responsible for a finalizer later in the list, resulting in a deadlock. Without enforced ordering finalizers are free to order amongst themselves and are not vulnerable to ordering changes in the list.\",\"items\":{\"default\":\"\",\"type\":\"string\"},\"type\":\"array\",\"x-kubernetes-list-type\":\"set\",\"x-kubernetes-patch-strategy\":\"merge\"},\"generateName\":{\"description\":\"GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\\n\\nIf this field is specified and the generated name exists, the server will return a 409.\\n\\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency\",\"type\":\"string\"},\"generation\":{\"description\":\"A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.\",\"format\":\"int64\",\"type\":\"integer\"},\"labels\":{\"additionalProperties\":{\"default\":\"\",\"type\":\"string\"},\"description\":\"Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels\",\"type\":\"object\"},\"managedFields\":{\"description\":\"ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \\\"ci-cd\\\". The set of fields is always in the version that the workflow used when modifying the object.\",\"items\":{\"allOf\":[{\"description\":\"ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.\",\"properties\":{\"apiVersion\":{\"description\":\"APIVersion defines the version of this resource that this field set applies to. The format is \\\"group/version\\\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.\",\"type\":\"string\"},\"fieldsType\":{\"description\":\"FieldsType is the discriminator for the different fields format and version. There is currently only one possible value: \\\"FieldsV1\\\"\",\"type\":\"string\"},\"fieldsV1\":{\"allOf\":[{\"description\":\"FieldsV1 stores a set of fields in a data structure like a Trie, in JSON format.\\n\\nEach key is either a '.' representing the field itself, and will always map to an empty set, or a string representing a sub-field or item. The string will follow one of these four formats: 'f:\\u003cname\\u003e', where \\u003cname\\u003e is the name of a field in a struct, or key in a map 'v:\\u003cvalue\\u003e', where \\u003cvalue\\u003e is the exact json formatted value of a list item 'i:\\u003cindex\\u003e', where \\u003cindex\\u003e is position of a item in a list 'k:\\u003ckeys\\u003e', where \\u003ckeys\\u003e is a map of a list item's key fields to their unique values If a key maps to an empty Fields value, the field that key represents is part of the set.\\n\\nThe exact format is defined in sigs.k8s.io/structured-merge-diff\",\"type\":\"object\"}],\"description\":\"FieldsV1 holds the first JSON version format as described in the \\\"FieldsV1\\\" type.\"},\"manager\":{\"description\":\"Manager is an identifier of the workflow managing these fields.\",\"type\":\"string\"},\"operation\":{\"description\":\"Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.\",\"type\":\"string\"},\"subresource\":{\"description\":\"Subresource is the name of the subresource used to update that object, or empty string if the object was updated through the main resource. The value of this field is used to distinguish between managers, even if they share the same name. For example, a status update will be distinct from a regular update using the same manager name. Note that the APIVersion field is not related to the Subresource field and it always corresponds to the version of the main resource.\",\"type\":\"string\"},\"time\":{\"allOf\":[{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"}],\"description\":\"Time is the timestamp of when the ManagedFields entry was added. The timestamp will also be updated if a field is added, the manager changes any of the owned fields value or removes a field. The timestamp does not update when a field is removed from the entry because another manager took it over.\"}},\"type\":\"object\"}],\"default\":{}},\"type\":\"array\",\"x-kubernetes-list-type\":\"atomic\"},\"name\":{\"description\":\"Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names\",\"type\":\"string\"},\"namespace\":{\"description\":\"Namespace defines the space within which each name must be unique. An empty namespace is equivalent to the \\\"default\\\" namespace, but \\\"default\\\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\\n\\nMust be a DNS_LABEL. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces\",\"type\":\"string\"},\"ownerReferences\":{\"description\":\"List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.\",\"items\":{\"allOf\":[{\"description\":\"OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.\",\"properties\":{\"apiVersion\":{\"default\":\"\",\"description\":\"API version of the referent.\",\"type\":\"string\"},\"blockOwnerDeletion\":{\"description\":\"If true, AND if the owner has the \\\"foregroundDeletion\\\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. See https://kubernetes.io/docs/concepts/architecture/garbage-collection/#foreground-deletion for how the garbage collector interacts with this field and enforces the foreground deletion. Defaults to false. To set this field, a user needs \\\"delete\\\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.\",\"type\":\"boolean\"},\"controller\":{\"description\":\"If true, this reference points to the managing controller.\",\"type\":\"boolean\"},\"kind\":{\"default\":\"\",\"description\":\"Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"default\":\"\",\"description\":\"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names\",\"type\":\"string\"},\"uid\":{\"default\":\"\",\"description\":\"UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#uids\",\"type\":\"string\"}},\"required\":[\"apiVersion\",\"kind\",\"name\",\"uid\"],\"type\":\"object\",\"x-kubernetes-map-type\":\"atomic\"}],\"default\":{}},\"type\":\"array\",\"x-kubernetes-list-map-keys\":[\"uid\"],\"x-kubernetes-list-type\":\"map\",\"x-kubernetes-patch-merge-key\":\"uid\",\"x-kubernetes-patch-strategy\":\"merge\"},\"resourceVersion\":{\"description\":\"An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\\n\\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency\",\"type\":\"string\"},\"selfLink\":{\"description\":\"Deprecated: selfLink is a legacy read-only field that is no longer populated by the system.\",\"type\":\"string\"},\"uid\":{\"description\":\"UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\\n\\nPopulated by the system. Read-only. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#uids\",\"type\":\"string\"}},\"type\":\"object\"}],\"default\":{},\"description\":\"Standard object metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata\"},\"spec\":{\"allOf\":[{\"description\":\"specification of a horizontal pod autoscaler.\",\"properties\":{\"maxReplicas\":{\"default\":0,\"description\":\"maxReplicas is the upper limit for the number of pods that can be set by the autoscaler; cannot be smaller than MinReplicas.\",\"format\":\"int32\",\"type\":\"integer\"},\"minReplicas\":{\"description\":\"minReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. It defaults to 1 pod. minReplicas is allowed to be 0 if the alpha feature gate HPAScaleToZero is enabled and at least one Object or External metric is configured. Scaling is active as long as at least one metric value is available.\",\"format\":\"int32\",\"type\":\"integer\"},\"scaleTargetRef\":{\"allOf\":[{\"description\":\"CrossVersionObjectReference contains enough information to let you identify the referred resource.\",\"properties\":{\"apiVersion\":{\"description\":\"apiVersion is the API version of the referent\",\"type\":\"string\"},\"kind\":{\"default\":\"\",\"description\":\"kind is the kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"default\":\"\",\"description\":\"name is the name of the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\",\"type\":\"string\"}},\"required\":[\"kind\",\"name\"],\"type\":\"object\",\"x-kubernetes-map-type\":\"atomic\"}],\"default\":{},\"description\":\"reference to scaled resource; horizontal pod autoscaler will learn the current resource consumption and will set the desired number of pods by using its Scale subresource.\"},\"targetCPUUtilizationPercentage\":{\"description\":\"targetCPUUtilizationPercentage is the target average CPU utilization (represented as a percentage of requested CPU) over all the pods; if not specified the default autoscaling policy will be used.\",\"format\":\"int32\",\"type\":\"integer\"}},\"required\":[\"scaleTargetRef\",\"maxReplicas\"],\"type\":\"object\"}],\"default\":{},\"description\":\"spec defines the behaviour of autoscaler. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status.\"},\"status\":{\"allOf\":[{\"description\":\"current status of a horizontal pod autoscaler\",\"properties\":{\"currentCPUUtilizationPercentage\":{\"description\":\"currentCPUUtilizationPercentage is the current average CPU utilization over all pods, represented as a percentage of requested CPU, e.g. 70 means that an average pod is using now 70% of its requested CPU.\",\"format\":\"int32\",\"type\":\"integer\"},\"currentReplicas\":{\"default\":0,\"description\":\"currentReplicas is the current number of replicas of pods managed by this autoscaler.\",\"format\":\"int32\",\"type\":\"integer\"},\"desiredReplicas\":{\"default\":0,\"description\":\"desiredReplicas is the desired number of replicas of pods managed by this autoscaler.\",\"format\":\"int32\",\"type\":\"integer\"},\"lastScaleTime\":{\"allOf\":[{\"description\":\"Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.\",\"format\":\"date-time\",\"type\":\"string\"}],\"description\":\"lastScaleTime is the last time the HorizontalPodAutoscaler scaled the number of pods; used by the autoscaler to control how often the number of pods is changed.\"},\"observedGeneration\":{\"description\":\"observedGeneration is the most recent generation observed by this autoscaler.\",\"format\":\"int64\",\"type\":\"integer\"}},\"required\":[\"currentReplicas\",\"desiredReplicas\"],\"type\":\"object\"}],\"default\":{},\"description\":\"status is the current information about the autoscaler.\"}},\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"autoscaling\",\"kind\":\"HorizontalPodAutoscaler\",\"version\":\"v1\"}]}],\"default\":{}},\"type\":\"array\"}},\"required\":[\"items\"],\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"autoscaling\",\"kind\":\"HorizontalPodAutoscalerList\",\"version\":\"v1\"}]}", -"version": "autoscaling/v1" -}, -"configuration": null, -"description": "", -"displayName": "horizontalpodautoscalerlist", -"format": "JSON", -"id": "00000000-0000-0000-0000-000000000000", -"metadata": { -"genealogy": "", -"isAnnotation": false, -"isNamespaced": false, -"published": false -}, -"model": { -"category": { -"name": "Orchestration \u0026 Management" -}, -"displayName": "Kubernetes", -"id": "00000000-0000-0000-0000-000000000000", -"metadata": { -"source_uri": "https://raw.githubusercontent.com/kubernetes/kubernetes/refs/heads/master/api/openapi-spec/v3/apis__autoscaling__v1_openapi.json/v1.31.3", -"svgColor": "", -"svgWhite": "" -}, -"model": { -"version": "version" -}, -"name": "kubernetes", -"registrant": { -"created_at": "0001-01-01T00:00:00Z", -"credential_id": "00000000-0000-0000-0000-000000000000", -"deleted_at": "0001-01-01T00:00:00Z", -"id": "00000000-0000-0000-0000-000000000000", -"kind": "", -"name": "", -"status": "", -"sub_type": "", -"type": "", -"updated_at": "0001-01-01T00:00:00Z", -"user_id": "00000000-0000-0000-0000-000000000000" -}, -"connection_id": "00000000-0000-0000-0000-000000000000", -"schemaVersion": "models.meshery.io/v1beta1", -"status": "", -"version": "v1.0.0", -"components": null, -"relationships": null -}, -"schemaVersion": "components.meshery.io/v1beta1", -"status": null, -"styles": null, -"version": "v1.0.0" -} \ No newline at end of file diff --git a/generators/github/kubernetes/status.json b/generators/github/kubernetes/status.json deleted file mode 100644 index 2eef9691..00000000 --- a/generators/github/kubernetes/status.json +++ /dev/null @@ -1,57 +0,0 @@ -{ -"component": { -"kind": "status", -"schema": "{\"description\":\"Status is a return value for calls that don't return other objects.\",\"properties\":{\"code\":{\"description\":\"Suggested HTTP return code for this status, 0 if not set.\",\"format\":\"int32\",\"type\":\"integer\"},\"details\":{\"allOf\":[{\"description\":\"StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.\",\"properties\":{\"causes\":{\"description\":\"The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.\",\"items\":{\"allOf\":[{\"description\":\"StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.\",\"properties\":{\"field\":{\"description\":\"The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\\n\\nExamples:\\n \\\"name\\\" - the field \\\"name\\\" on the current resource\\n \\\"items[0].name\\\" - the field \\\"name\\\" on the first array entry in \\\"items\\\"\",\"type\":\"string\"},\"message\":{\"description\":\"A human-readable description of the cause of the error. This field may be presented as-is to a reader.\",\"type\":\"string\"},\"reason\":{\"description\":\"A machine-readable description of the cause of the error. If this value is empty there is no information available.\",\"type\":\"string\"}},\"type\":\"object\"}],\"default\":{}},\"type\":\"array\",\"x-kubernetes-list-type\":\"atomic\"},\"group\":{\"description\":\"The group attribute of the resource associated with the status StatusReason.\",\"type\":\"string\"},\"kind\":{\"description\":\"The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\",\"type\":\"string\"},\"name\":{\"description\":\"The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).\",\"type\":\"string\"},\"retryAfterSeconds\":{\"description\":\"If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.\",\"format\":\"int32\",\"type\":\"integer\"},\"uid\":{\"description\":\"UID of the resource. (when there is a single resource which can be described). More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#uids\",\"type\":\"string\"}},\"type\":\"object\"}],\"description\":\"Extended data associated with the reason. Each reason may define its own extended details. This field is optional and the data returned is not guaranteed to conform to any schema except that defined by the reason type.\",\"x-kubernetes-list-type\":\"atomic\"},\"message\":{\"description\":\"A human-readable description of the status of this operation.\",\"type\":\"string\"},\"reason\":{\"description\":\"A machine-readable description of why this operation is in the \\\"Failure\\\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.\",\"type\":\"string\"}},\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"\",\"kind\":\"Status\",\"version\":\"v1\"}]}", -"version": "v1" -}, -"configuration": null, -"description": "", -"displayName": "status", -"format": "JSON", -"id": "00000000-0000-0000-0000-000000000000", -"metadata": { -"genealogy": "", -"isAnnotation": false, -"isNamespaced": false, -"published": false -}, -"model": { -"category": { -"name": "Orchestration \u0026 Management" -}, -"displayName": "Kubernetes", -"id": "00000000-0000-0000-0000-000000000000", -"metadata": { -"source_uri": "https://raw.githubusercontent.com/kubernetes/kubernetes/refs/heads/master/api/openapi-spec/v3/apis__autoscaling__v1_openapi.json/v1.31.3", -"svgColor": "", -"svgWhite": "" -}, -"model": { -"version": "version" -}, -"name": "kubernetes", -"registrant": { -"created_at": "0001-01-01T00:00:00Z", -"credential_id": "00000000-0000-0000-0000-000000000000", -"deleted_at": "0001-01-01T00:00:00Z", -"id": "00000000-0000-0000-0000-000000000000", -"kind": "", -"name": "", -"status": "", -"sub_type": "", -"type": "", -"updated_at": "0001-01-01T00:00:00Z", -"user_id": "00000000-0000-0000-0000-000000000000" -}, -"connection_id": "00000000-0000-0000-0000-000000000000", -"schemaVersion": "models.meshery.io/v1beta1", -"status": "", -"version": "v1.0.0", -"components": null, -"relationships": null -}, -"schemaVersion": "components.meshery.io/v1beta1", -"status": null, -"styles": null, -"version": "v1.0.0" -} \ No newline at end of file diff --git a/generators/github/kubernetes/watchevent.json b/generators/github/kubernetes/watchevent.json deleted file mode 100644 index 342c16f8..00000000 --- a/generators/github/kubernetes/watchevent.json +++ /dev/null @@ -1,57 +0,0 @@ -{ -"component": { -"kind": "watchevent", -"schema": "{\"description\":\"Event represents a single event to a watched resource.\",\"properties\":{\"object\":{\"allOf\":[{\"description\":\"RawExtension is used to hold extensions in external versions.\\n\\nTo use this, make a field which has RawExtension as its type in your external, versioned struct, and Object in your internal struct. You also need to register your various plugin types.\\n\\n// Internal package:\\n\\n\\ttype MyAPIObject struct {\\n\\t\\truntime.TypeMeta `json:\\\",inline\\\"`\\n\\t\\tMyPlugin runtime.Object `json:\\\"myPlugin\\\"`\\n\\t}\\n\\n\\ttype PluginA struct {\\n\\t\\tAOption string `json:\\\"aOption\\\"`\\n\\t}\\n\\n// External package:\\n\\n\\ttype MyAPIObject struct {\\n\\t\\truntime.TypeMeta `json:\\\",inline\\\"`\\n\\t\\tMyPlugin runtime.RawExtension `json:\\\"myPlugin\\\"`\\n\\t}\\n\\n\\ttype PluginA struct {\\n\\t\\tAOption string `json:\\\"aOption\\\"`\\n\\t}\\n\\n// On the wire, the JSON will look something like this:\\n\\n\\t{\\n\\t\\t\\\"kind\\\":\\\"MyAPIObject\\\",\\n\\t\\t\\\"apiVersion\\\":\\\"v1\\\",\\n\\t\\t\\\"myPlugin\\\": {\\n\\t\\t\\t\\\"kind\\\":\\\"PluginA\\\",\\n\\t\\t\\t\\\"aOption\\\":\\\"foo\\\",\\n\\t\\t},\\n\\t}\\n\\nSo what happens? Decode first uses json or yaml to unmarshal the serialized data into your external MyAPIObject. That causes the raw JSON to be stored, but not unpacked. The next step is to copy (using pkg/conversion) into the internal struct. The runtime package's DefaultScheme has conversion functions installed which will unpack the JSON stored in RawExtension, turning it into the correct object type, and storing it in the Object. (TODO: In the case where the object is of an unknown type, a runtime.Unknown object will be created and stored.)\",\"type\":\"object\"}],\"description\":\"Object is:\\n * If Type is Added or Modified: the new state of the object.\\n * If Type is Deleted: the state of the object immediately before deletion.\\n * If Type is Error: *Status is recommended; other types may make sense\\n depending on context.\"},\"type\":{\"default\":\"\",\"type\":\"string\"}},\"required\":[\"type\",\"object\"],\"type\":\"object\",\"x-kubernetes-group-version-kind\":[{\"group\":\"\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"admission.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"admission.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"admissionregistration.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"admissionregistration.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"admissionregistration.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"apiextensions.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"apiextensions.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"apiregistration.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"apiregistration.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"apps\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"apps\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"apps\",\"kind\":\"WatchEvent\",\"version\":\"v1beta2\"},{\"group\":\"authentication.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"authentication.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"authentication.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"authorization.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"authorization.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"autoscaling\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"autoscaling\",\"kind\":\"WatchEvent\",\"version\":\"v2\"},{\"group\":\"autoscaling\",\"kind\":\"WatchEvent\",\"version\":\"v2beta1\"},{\"group\":\"autoscaling\",\"kind\":\"WatchEvent\",\"version\":\"v2beta2\"},{\"group\":\"batch\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"batch\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"certificates.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"certificates.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"certificates.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"coordination.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"coordination.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"coordination.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"discovery.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"discovery.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"events.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"events.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"extensions\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta2\"},{\"group\":\"flowcontrol.apiserver.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta3\"},{\"group\":\"imagepolicy.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"internal.apiserver.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"networking.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"networking.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"networking.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"node.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"node.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"node.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"policy\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"policy\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"rbac.authorization.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"rbac.authorization.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"rbac.authorization.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"resource.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha3\"},{\"group\":\"scheduling.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"scheduling.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"scheduling.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"storage.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1\"},{\"group\":\"storage.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"},{\"group\":\"storage.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1beta1\"},{\"group\":\"storagemigration.k8s.io\",\"kind\":\"WatchEvent\",\"version\":\"v1alpha1\"}]}", -"version": "v1" -}, -"configuration": null, -"description": "", -"displayName": "watchevent", -"format": "JSON", -"id": "00000000-0000-0000-0000-000000000000", -"metadata": { -"genealogy": "", -"isAnnotation": false, -"isNamespaced": false, -"published": false -}, -"model": { -"category": { -"name": "Orchestration \u0026 Management" -}, -"displayName": "Kubernetes", -"id": "00000000-0000-0000-0000-000000000000", -"metadata": { -"source_uri": "https://raw.githubusercontent.com/kubernetes/kubernetes/refs/heads/master/api/openapi-spec/v3/apis__autoscaling__v1_openapi.json/v1.31.3", -"svgColor": "", -"svgWhite": "" -}, -"model": { -"version": "version" -}, -"name": "kubernetes", -"registrant": { -"created_at": "0001-01-01T00:00:00Z", -"credential_id": "00000000-0000-0000-0000-000000000000", -"deleted_at": "0001-01-01T00:00:00Z", -"id": "00000000-0000-0000-0000-000000000000", -"kind": "", -"name": "", -"status": "", -"sub_type": "", -"type": "", -"updated_at": "0001-01-01T00:00:00Z", -"user_id": "00000000-0000-0000-0000-000000000000" -}, -"connection_id": "00000000-0000-0000-0000-000000000000", -"schemaVersion": "models.meshery.io/v1beta1", -"status": "", -"version": "v1.0.0", -"components": null, -"relationships": null -}, -"schemaVersion": "components.meshery.io/v1beta1", -"status": null, -"styles": null, -"version": "v1.0.0" -} \ No newline at end of file From 45521cd6e072dba4e2bd0e989e1724a7a860e842 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 30 Sep 2024 15:41:41 +0530 Subject: [PATCH 096/112] finalize changes Signed-off-by: MUzairS15 --- utils/component/generator.go | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/utils/component/generator.go b/utils/component/generator.go index 0b974245..b228b363 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -7,7 +7,6 @@ import ( "cuelang.org/go/cue" "github.com/layer5io/meshkit/utils" - "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1" "github.com/meshery/schemas/models/v1beta1/component" @@ -63,38 +62,28 @@ func Generate(resource string) (component.ComponentDefinition, error) { cmp.SchemaVersion = v1beta1.ComponentSchemaVersion cmp.Metadata = component.ComponentDefinition_Metadata{} - isCRD := kubernetes.IsCRD(resource) - - cueValue, err := cueValueFromResource(resource, isCRD) + crdCue, err := utils.YamlToCue(resource) if err != nil { return cmp, err } - var specPath string - if isCRD { - specPath = DefaultPathConfig.SpecPath - } else { - specPath = "components.schemas" - } - var schema string for _, cfg := range Configs { - cfg.SpecPath = specPath - schema, err = getSchema(cueValue, cfg) + schema, err = getSchema(crdCue, cfg) if err == nil { break } } cmp.Component.Schema = schema - name, err := extractCueValueFromPath(cueValue, DefaultPathConfig.NamePath) + name, err := extractCueValueFromPath(crdCue, DefaultPathConfig.NamePath) if err != nil { return cmp, err } - version, err := extractCueValueFromPath(cueValue, DefaultPathConfig.VersionPath) + version, err := extractCueValueFromPath(crdCue, DefaultPathConfig.VersionPath) if err != nil { return cmp, err } - group, err := extractCueValueFromPath(cueValue, DefaultPathConfig.GroupPath) + group, err := extractCueValueFromPath(crdCue, DefaultPathConfig.GroupPath) if err != nil { return cmp, err } @@ -102,7 +91,7 @@ func Generate(resource string) (component.ComponentDefinition, error) { if cmp.Metadata.AdditionalProperties == nil { cmp.Metadata.AdditionalProperties = make(map[string]interface{}) } - scope, _ := extractCueValueFromPath(cueValue, DefaultPathConfig.ScopePath) + scope, _ := extractCueValueFromPath(crdCue, DefaultPathConfig.ScopePath) if scope == "Cluster" { cmp.Metadata.IsNamespaced = false } else if scope == "Namespaced" { @@ -120,14 +109,6 @@ func Generate(resource string) (component.ComponentDefinition, error) { return cmp, nil } -func cueValueFromResource(resource string, isCRD bool) (cue.Value, error) { - if isCRD { - return utils.YamlToCue(resource) - } else { - return utils.JsonToCue([]byte(resource)) - } -} - /* Find and modify specific schema properties. 1. Identify interesting properties by walking entire schema. From 98a1e99e5fc1bf7874de9a9557b8558a2ab8e61b Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 30 Sep 2024 15:53:31 +0530 Subject: [PATCH 097/112] finalize changes Signed-off-by: MUzairS15 --- generators/github/package.go | 2 +- utils/component/openapi_generator.go | 28 ++++++++++++++++++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/generators/github/package.go b/generators/github/package.go index aff3be4b..e7efea6c 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -51,7 +51,7 @@ func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, comps, err := component.GenerateFromOpenAPI(string(crd), gp) if err != nil { - errs = append(errs, err) + errs = append(errs, component.ErrGetSchema(err)) continue } components = append(components, comps...) diff --git a/utils/component/openapi_generator.go b/utils/component/openapi_generator.go index 12d37a59..f1acf67d 100644 --- a/utils/component/openapi_generator.go +++ b/utils/component/openapi_generator.go @@ -30,8 +30,12 @@ func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.Compo } cuectx := cuecontext.New() cueParsedManExpr, err := cueJson.Extract("", []byte(resource)) + if err != nil { + return nil, err + } + parsedManifest := cuectx.BuildExpr(cueParsedManExpr) - definitions := parsedManifest.LookupPath(cue.ParsePath("components.schemas")) + definitions, err := utils.Lookup(parsedManifest, "components.schemas") if err != nil { return nil, err @@ -46,8 +50,8 @@ func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.Compo for fields.Next() { fieldVal := fields.Value() - kindCue := fieldVal.LookupPath(cue.ParsePath(`"x-kubernetes-group-version-kind"[0].kind`)) - if kindCue.Err() != nil { + kindCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].kind`) + if err != nil { continue } kind, err := kindCue.String() @@ -62,8 +66,16 @@ func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.Compo fmt.Printf("%v", err) continue } - versionCue := fieldVal.LookupPath(cue.ParsePath(`"x-kubernetes-group-version-kind"[0].version`)) - groupCue := fieldVal.LookupPath(cue.ParsePath(`"x-kubernetes-group-version-kind"[0].group`)) + versionCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].version`) + if err != nil { + continue + } + + groupCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].group`) + if err != nil { + continue + } + apiVersion, _ := versionCue.String() if g, _ := groupCue.String(); g != "" { apiVersion = g + "/" + apiVersion @@ -140,8 +152,12 @@ func getResolvedManifest(manifest string) (string, error) { cuectx := cuecontext.New() cueParsedManExpr, err := cueJson.Extract("", byt) + if err != nil { + return "", ErrGetSchema(err) + } + parsedManifest := cuectx.BuildExpr(cueParsedManExpr) - definitions := parsedManifest.LookupPath(cue.ParsePath("components.schemas")) + definitions, err := utils.Lookup(parsedManifest, "components.schemas") if err != nil { return "", err } From d216a0a29ec0ee5a0f52c3ec52df8e27c987a526 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Mon, 30 Sep 2024 18:14:49 +0530 Subject: [PATCH 098/112] utility function to get latest Version w/signoff Signed-off-by: Jougan-0 --- utils/utils.go | 128 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/utils/utils.go b/utils/utils.go index 77b2dd64..737ca1c0 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -17,8 +17,10 @@ import ( "reflect" "regexp" "runtime" + "sort" "strconv" "strings" + "unicode" "github.com/layer5io/meshkit/models/meshmodel/entity" log "github.com/sirupsen/logrus" @@ -586,3 +588,129 @@ func Compress(src string, buf io.Writer) error { return nil }) } + +// Check if a string is purely numeric +func isNumeric(s string) bool { + for _, r := range s { + if !unicode.IsDigit(r) { + return false + } + } + return true +} + +// Split version into components (numeric and non-numeric) using both '.' and '-' +func splitVersion(version string) []string { + version = strings.ReplaceAll(version, "-", ".") + return strings.Split(version, ".") +} + +// Compare two version strings +func compareVersions(v1, v2 string) int { + v1Components := splitVersion(v1) + v2Components := splitVersion(v2) + + maxLen := len(v1Components) + if len(v2Components) > maxLen { + maxLen = len(v2Components) + } + + for i := 0; i < maxLen; i++ { + var part1, part2 string + if i < len(v1Components) { + part1 = v1Components[i] + } + if i < len(v2Components) { + part2 = v2Components[i] + } + + if isNumeric(part1) && isNumeric(part2) { + num1, _ := strconv.Atoi(part1) + num2, _ := strconv.Atoi(part2) + if num1 != num2 { + return num1 - num2 + } + } else if isNumeric(part1) && !isNumeric(part2) { + return -1 + } else if !isNumeric(part1) && isNumeric(part2) { + return 1 + } else { + if part1 != part2 { + return strings.Compare(part1, part2) + } + } + } + + return 0 +} + +// Function to get all version directories sorted in descending order +func GetAllVersionDirsSortedDesc(modelVersionsDirPath string) ([]string, error) { + type versionInfo struct { + original string + dirPath string + } + entries, err := os.ReadDir(modelVersionsDirPath) + if err != nil { + return nil, fmt.Errorf("failed to read versions directory '%s': %w", modelVersionsDirPath, err) + } + + if len(entries) == 0 { + return nil, fmt.Errorf("no version directories found in '%s'", modelVersionsDirPath) + } + + versions := []versionInfo{} + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + versionDirPath := filepath.Join(modelVersionsDirPath, entry.Name()) + versionStr := entry.Name() + + // Optionally remove leading 'v' + versionStr = strings.TrimPrefix(versionStr, "v") + + if versionStr == "" { + continue + } + + versions = append(versions, versionInfo{ + original: versionStr, + dirPath: versionDirPath, + }) + } + + if len(versions) == 0 { + return nil, fmt.Errorf("no valid version directories found in '%s'", modelVersionsDirPath) + } + + sort.Slice(versions, func(i, j int) bool { + return compareVersions(versions[i].original, versions[j].original) > 0 + }) + + sortedDirPaths := make([]string, len(versions)) + for i, v := range versions { + sortedDirPaths[i] = v.dirPath + } + + return sortedDirPaths, nil +} + +// isDirectoryNonEmpty checks if a directory exists and is non-empty +func IsDirectoryNonEmpty(dirPath string) bool { + fi, err := os.Stat(dirPath) + if err != nil { + return false + } + if !fi.IsDir() { + return false + } + + entries, err := os.ReadDir(dirPath) + if err != nil { + return false + } + + return len(entries) > 0 +} From d889a68359a1f9de0462a53e06ca21c9aca62cc1 Mon Sep 17 00:00:00 2001 From: Lee Calcote Date: Mon, 30 Sep 2024 16:24:54 -0500 Subject: [PATCH 099/112] Revert "support generation of model from openapi schemas" --- encoding/convert.go | 15 --- generators/artifacthub/package.go | 8 -- generators/github/package.go | 59 +++------ generators/models/interfaces.go | 2 - utils/component/generator.go | 15 +-- utils/component/openapi_generator.go | 172 --------------------------- utils/component/utils.go | 16 ++- utils/helm/helm.go | 9 +- utils/kubernetes/crd.go | 19 +-- utils/manifests/utils.go | 15 ++- 10 files changed, 45 insertions(+), 285 deletions(-) delete mode 100644 encoding/convert.go delete mode 100644 utils/component/openapi_generator.go diff --git a/encoding/convert.go b/encoding/convert.go deleted file mode 100644 index 5dc810eb..00000000 --- a/encoding/convert.go +++ /dev/null @@ -1,15 +0,0 @@ -package encoding - -import ( - "gopkg.in/yaml.v3" -) - -func ToYaml(data []byte) ([]byte, error) { - var out map[string]interface{} - err := Unmarshal(data, &out) - if err != nil { - return nil, err - } - - return yaml.Marshal(out) -} diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index e40509d8..61b3d8be 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -38,14 +38,6 @@ func (pkg AhPackage) GetVersion() string { return pkg.Version } -func (pkg AhPackage) GetSourceURL() string { - return pkg.ChartUrl -} - -func (pkg AhPackage) GetName() string { - return pkg.Name -} - func (pkg AhPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { components := make([]_component.ComponentDefinition, 0) // TODO: Move this to the configuration diff --git a/generators/github/package.go b/generators/github/package.go index e7efea6c..0c632ef9 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -6,7 +6,6 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" - "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1/category" _component "github.com/meshery/schemas/models/v1beta1/component" @@ -26,14 +25,6 @@ func (gp GitHubPackage) GetVersion() string { return gp.version } -func (gp GitHubPackage) GetSourceURL() string { - return gp.SourceURL -} - -func (gp GitHubPackage) GetName() string { - return gp.Name -} - func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { components := make([]_component.ComponentDefinition, 0) @@ -43,40 +34,28 @@ func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, } manifestBytes := bytes.Split(data, []byte("\n---\n")) - errs := []error{} - - for _, crd := range manifestBytes { - isCrd := kubernetes.IsCRD(string(crd)) - if !isCrd { + crds, errs := component.FilterCRDs(manifestBytes) - comps, err := component.GenerateFromOpenAPI(string(crd), gp) - if err != nil { - errs = append(errs, component.ErrGetSchema(err)) - continue - } - components = append(components, comps...) - } else { - comp, err := component.Generate(string(crd)) - if err != nil { - continue - } - if comp.Model.Metadata == nil { - comp.Model.Metadata = &model.ModelDefinition_Metadata{} - } - if comp.Model.Metadata.AdditionalProperties == nil { - comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) - } - - comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL - comp.Model.Version = gp.version - comp.Model.Name = gp.Name - comp.Model.Category = category.CategoryDefinition{ - Name: "", - } - comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) - components = append(components, comp) + for _, crd := range crds { + comp, err := component.Generate(crd) + if err != nil { + continue + } + if comp.Model.Metadata == nil { + comp.Model.Metadata = &model.ModelDefinition_Metadata{} + } + if comp.Model.Metadata.AdditionalProperties == nil { + comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) } + comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL + comp.Model.Version = gp.version + comp.Model.Name = gp.Name + comp.Model.Category = category.CategoryDefinition{ + Name: "", + } + comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) + components = append(components, comp) } return components, utils.CombineErrors(errs, "\n") diff --git a/generators/models/interfaces.go b/generators/models/interfaces.go index c6dcd3eb..19f61177 100644 --- a/generators/models/interfaces.go +++ b/generators/models/interfaces.go @@ -13,8 +13,6 @@ type Validator interface { type Package interface { GenerateComponents() ([]component.ComponentDefinition, error) GetVersion() string - GetSourceURL() string - GetName() string } // Supports pulling packages from Artifact Hub and other sources like Docker Hub. diff --git a/utils/component/generator.go b/utils/component/generator.go index b228b363..288f2780 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -45,28 +45,17 @@ var DefaultPathConfig2 = CuePathConfig{ SpecPath: "spec.validation.openAPIV3Schema", } -var OpenAPISpecPathConfig = CuePathConfig{ - NamePath: `x-kubernetes-group-version-kind"[0].kind`, - IdentifierPath: "spec.names.kind", - VersionPath: `"x-kubernetes-group-version-kind"[0].version`, - GroupPath: `"x-kubernetes-group-version-kind"[0].group`, - ScopePath: "spec.scope", - SpecPath: "spec.versions[0].schema.openAPIV3Schema", - PropertiesPath: "properties", -} - var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2} -func Generate(resource string) (component.ComponentDefinition, error) { +func Generate(crd string) (component.ComponentDefinition, error) { cmp := component.ComponentDefinition{} cmp.SchemaVersion = v1beta1.ComponentSchemaVersion cmp.Metadata = component.ComponentDefinition_Metadata{} - crdCue, err := utils.YamlToCue(resource) + crdCue, err := utils.YamlToCue(crd) if err != nil { return cmp, err } - var schema string for _, cfg := range Configs { schema, err = getSchema(crdCue, cfg) diff --git a/utils/component/openapi_generator.go b/utils/component/openapi_generator.go deleted file mode 100644 index f1acf67d..00000000 --- a/utils/component/openapi_generator.go +++ /dev/null @@ -1,172 +0,0 @@ -package component - -import ( - "encoding/json" - "fmt" - "strings" - - "cuelang.org/go/cue" - "cuelang.org/go/cue/cuecontext" - cueJson "cuelang.org/go/encoding/json" - "github.com/layer5io/meshkit/generators/models" - "github.com/layer5io/meshkit/utils" - "github.com/layer5io/meshkit/utils/manifests" - - "gopkg.in/yaml.v3" - - "github.com/meshery/schemas/models/v1beta1" - "github.com/meshery/schemas/models/v1beta1/category" - "github.com/meshery/schemas/models/v1beta1/component" - "github.com/meshery/schemas/models/v1beta1/model" -) - -func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.ComponentDefinition, error) { - if resource == "" { - return nil, nil - } - resource, err := getResolvedManifest(resource) - if err != nil { - return nil, err - } - cuectx := cuecontext.New() - cueParsedManExpr, err := cueJson.Extract("", []byte(resource)) - if err != nil { - return nil, err - } - - parsedManifest := cuectx.BuildExpr(cueParsedManExpr) - definitions, err := utils.Lookup(parsedManifest, "components.schemas") - - if err != nil { - return nil, err - } - - fields, err := definitions.Fields() - if err != nil { - fmt.Printf("%v\n", err) - return nil, err - } - components := make([]component.ComponentDefinition, 0) - - for fields.Next() { - fieldVal := fields.Value() - kindCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].kind`) - if err != nil { - continue - } - kind, err := kindCue.String() - kind = strings.ToLower(kind) - if err != nil { - fmt.Printf("%v", err) - continue - } - - crd, err := fieldVal.MarshalJSON() - if err != nil { - fmt.Printf("%v", err) - continue - } - versionCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].version`) - if err != nil { - continue - } - - groupCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].group`) - if err != nil { - continue - } - - apiVersion, _ := versionCue.String() - if g, _ := groupCue.String(); g != "" { - apiVersion = g + "/" + apiVersion - } - modified := make(map[string]interface{}) //Remove the given fields which is either not required by End user (like status) or is prefilled by system (like apiVersion, kind and metadata) - err = json.Unmarshal(crd, &modified) - if err != nil { - fmt.Printf("%v", err) - continue - } - - modifiedProps, err := UpdateProperties(fieldVal, cue.ParsePath("properties.spec"), apiVersion) - if err == nil { - modified = modifiedProps - } - - DeleteFields(modified) - crd, err = json.Marshal(modified) - if err != nil { - fmt.Printf("%v", err) - continue - } - - c := component.ComponentDefinition{ - SchemaVersion: v1beta1.ComponentSchemaVersion, - Version: "v1.0.0", - - Format: component.JSON, - Component: component.Component{ - Kind: kind, - Version: apiVersion, - Schema: string(crd), - }, - // Metadata: compMetadata, - DisplayName: manifests.FormatToReadableString(kind), - Model: model.ModelDefinition{ - SchemaVersion: v1beta1.ModelSchemaVersion, - Version: "v1.0.0", - - Model: model.Model{ - Version: pkg.GetVersion(), - }, - Name: pkg.GetName(), - DisplayName: manifests.FormatToReadableString(pkg.GetName()), - Category: category.CategoryDefinition{ - Name: "Orchestration & Management", - }, - Metadata: &model.ModelDefinition_Metadata{ - AdditionalProperties: map[string]interface{}{ - "source_uri": pkg.GetSourceURL(), - }, - }, - }, - } - - components = append(components, c) - } - return components, nil - -} - -func getResolvedManifest(manifest string) (string, error) { - var m map[string]interface{} - - err := yaml.Unmarshal([]byte(manifest), &m) - if err != nil { - return "", utils.ErrDecodeYaml(err) - } - - byt, err := json.Marshal(m) - if err != nil { - return "", utils.ErrMarshal(err) - } - - cuectx := cuecontext.New() - cueParsedManExpr, err := cueJson.Extract("", byt) - if err != nil { - return "", ErrGetSchema(err) - } - - parsedManifest := cuectx.BuildExpr(cueParsedManExpr) - definitions, err := utils.Lookup(parsedManifest, "components.schemas") - if err != nil { - return "", err - } - resol := manifests.ResolveOpenApiRefs{} - cache := make(map[string][]byte) - resolved, err := resol.ResolveReferences(byt, definitions, cache) - if err != nil { - return "", err - } - manifest = string(resolved) - return manifest, nil -} diff --git a/utils/component/utils.go b/utils/component/utils.go index f04ded45..f63d6068 100644 --- a/utils/component/utils.go +++ b/utils/component/utils.go @@ -7,6 +7,7 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" + "gopkg.in/yaml.v2" ) // Remove the fields which is either not required by end user (like status) or is prefilled by system (like apiVersion, kind and metadata) @@ -80,10 +81,19 @@ func FilterCRDs(manifests [][]byte) ([]string, []error) { var errs []error var filteredManifests []string for _, m := range manifests { - isCrd := kubernetes.IsCRD(string(m)) - if isCrd { - filteredManifests = append(filteredManifests, string(m)) + + var crd map[string]interface{} + err := yaml.Unmarshal(m, &crd) + if err != nil { + errs = append(errs, err) + continue + } + + isCrd := kubernetes.IsCRD(crd) + if !isCrd { + continue } + filteredManifests = append(filteredManifests, string(m)) } return filteredManifests, errs } diff --git a/utils/helm/helm.go b/utils/helm/helm.go index 091e5902..4af65eed 100644 --- a/utils/helm/helm.go +++ b/utils/helm/helm.go @@ -9,7 +9,6 @@ import ( "regexp" "strings" - "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/utils" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" @@ -109,13 +108,7 @@ func writeToFile(w io.Writer, path string) error { if err != nil { return utils.ErrReadFile(err, path) } - - byt, err := encoding.ToYaml(data) - if err != nil { - return utils.ErrWriteFile(err, path) - } - - _, err = w.Write(byt) + _, err = w.Write(data) if err != nil { return utils.ErrWriteFile(err, path) } diff --git a/utils/kubernetes/crd.go b/utils/kubernetes/crd.go index 9bf8d991..0525a624 100644 --- a/utils/kubernetes/crd.go +++ b/utils/kubernetes/crd.go @@ -4,7 +4,6 @@ import ( "context" "github.com/layer5io/meshkit/encoding" - "github.com/layer5io/meshkit/utils" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" ) @@ -54,19 +53,7 @@ func GetGVRForCustomResources(crd *CRDItem) *schema.GroupVersionResource { } } -func IsCRD(manifest string) bool { - cueValue, err := utils.YamlToCue(manifest) - if err != nil { - return false - } - kind, err := utils.Lookup(cueValue, "kind") - if err != nil { - return false - } - kindStr, err := kind.String() - if err != nil { - return false - } - - return kindStr == "CustomResourceDefinition" +func IsCRD(manifest map[string]interface{}) bool { + kind, ok := manifest["kind"].(string) + return ok && kind == "CustomResourceDefinition" } diff --git a/utils/manifests/utils.go b/utils/manifests/utils.go index 2935f1c0..fb3990f7 100644 --- a/utils/manifests/utils.go +++ b/utils/manifests/utils.go @@ -9,7 +9,6 @@ import ( "strings" "cuelang.org/go/cue" - "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/models/oam/core/v1alpha1" ) @@ -255,7 +254,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue cache = make(map[string][]byte) } var val map[string]interface{} - err := encoding.Unmarshal(manifest, &val) + err := json.Unmarshal(manifest, &val) if err != nil { return nil, err } @@ -267,7 +266,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue if ro.isInsideJsonSchemaProps && (ref == JsonSchemaPropsRef) { // hack so that the UI doesn't crash val["$ref"] = "string" - marVal, errJson := encoding.Marshal(val) + marVal, errJson := json.Marshal(val) if errJson != nil { return manifest, nil } @@ -283,13 +282,13 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue newval = append(newval, v0) continue } - byt, _ := encoding.Marshal(v0) + byt, _ := json.Marshal(v0) byt, err = ro.ResolveReferences(byt, definitions, cache) if err != nil { return nil, err } var newvalmap map[string]interface{} - _ = encoding.Unmarshal(byt, &newvalmap) + _ = json.Unmarshal(byt, &newvalmap) newval = append(newval, newvalmap) } val[k] = newval @@ -334,7 +333,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue if reflect.ValueOf(v).Kind() == reflect.Map { var marVal []byte var def []byte - marVal, err = encoding.Marshal(v) + marVal, err = json.Marshal(v) if err != nil { return nil, err } @@ -350,7 +349,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue } } } - res, err := encoding.Marshal(val) + res, err := json.Marshal(val) if err != nil { return nil, err } @@ -359,7 +358,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue func replaceRefWithVal(def []byte, val map[string]interface{}, k string) error { var defVal map[string]interface{} - err := encoding.Unmarshal([]byte(def), &defVal) + err := json.Unmarshal([]byte(def), &defVal) if err != nil { return err } From a60565401878181877257b27f7e8fdbe25cee82a Mon Sep 17 00:00:00 2001 From: Mohd Uzair Date: Thu, 3 Oct 2024 14:56:53 +0530 Subject: [PATCH 100/112] Revert "Revert "support generation of model from openapi schemas"" --- encoding/convert.go | 15 +++ generators/artifacthub/package.go | 8 ++ generators/github/package.go | 59 ++++++--- generators/models/interfaces.go | 2 + utils/component/generator.go | 15 ++- utils/component/openapi_generator.go | 172 +++++++++++++++++++++++++++ utils/component/utils.go | 16 +-- utils/helm/helm.go | 9 +- utils/kubernetes/crd.go | 19 ++- utils/manifests/utils.go | 15 +-- 10 files changed, 285 insertions(+), 45 deletions(-) create mode 100644 encoding/convert.go create mode 100644 utils/component/openapi_generator.go diff --git a/encoding/convert.go b/encoding/convert.go new file mode 100644 index 00000000..5dc810eb --- /dev/null +++ b/encoding/convert.go @@ -0,0 +1,15 @@ +package encoding + +import ( + "gopkg.in/yaml.v3" +) + +func ToYaml(data []byte) ([]byte, error) { + var out map[string]interface{} + err := Unmarshal(data, &out) + if err != nil { + return nil, err + } + + return yaml.Marshal(out) +} diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index 61b3d8be..e40509d8 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -38,6 +38,14 @@ func (pkg AhPackage) GetVersion() string { return pkg.Version } +func (pkg AhPackage) GetSourceURL() string { + return pkg.ChartUrl +} + +func (pkg AhPackage) GetName() string { + return pkg.Name +} + func (pkg AhPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { components := make([]_component.ComponentDefinition, 0) // TODO: Move this to the configuration diff --git a/generators/github/package.go b/generators/github/package.go index 0c632ef9..e7efea6c 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -6,6 +6,7 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" + "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1/category" _component "github.com/meshery/schemas/models/v1beta1/component" @@ -25,6 +26,14 @@ func (gp GitHubPackage) GetVersion() string { return gp.version } +func (gp GitHubPackage) GetSourceURL() string { + return gp.SourceURL +} + +func (gp GitHubPackage) GetName() string { + return gp.Name +} + func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { components := make([]_component.ComponentDefinition, 0) @@ -34,28 +43,40 @@ func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, } manifestBytes := bytes.Split(data, []byte("\n---\n")) - crds, errs := component.FilterCRDs(manifestBytes) + errs := []error{} - for _, crd := range crds { - comp, err := component.Generate(crd) - if err != nil { - continue - } - if comp.Model.Metadata == nil { - comp.Model.Metadata = &model.ModelDefinition_Metadata{} - } - if comp.Model.Metadata.AdditionalProperties == nil { - comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) - } + for _, crd := range manifestBytes { + isCrd := kubernetes.IsCRD(string(crd)) + if !isCrd { - comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL - comp.Model.Version = gp.version - comp.Model.Name = gp.Name - comp.Model.Category = category.CategoryDefinition{ - Name: "", + comps, err := component.GenerateFromOpenAPI(string(crd), gp) + if err != nil { + errs = append(errs, component.ErrGetSchema(err)) + continue + } + components = append(components, comps...) + } else { + comp, err := component.Generate(string(crd)) + if err != nil { + continue + } + if comp.Model.Metadata == nil { + comp.Model.Metadata = &model.ModelDefinition_Metadata{} + } + if comp.Model.Metadata.AdditionalProperties == nil { + comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) + } + + comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL + comp.Model.Version = gp.version + comp.Model.Name = gp.Name + comp.Model.Category = category.CategoryDefinition{ + Name: "", + } + comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) + components = append(components, comp) } - comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) - components = append(components, comp) + } return components, utils.CombineErrors(errs, "\n") diff --git a/generators/models/interfaces.go b/generators/models/interfaces.go index 19f61177..c6dcd3eb 100644 --- a/generators/models/interfaces.go +++ b/generators/models/interfaces.go @@ -13,6 +13,8 @@ type Validator interface { type Package interface { GenerateComponents() ([]component.ComponentDefinition, error) GetVersion() string + GetSourceURL() string + GetName() string } // Supports pulling packages from Artifact Hub and other sources like Docker Hub. diff --git a/utils/component/generator.go b/utils/component/generator.go index 288f2780..b228b363 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -45,17 +45,28 @@ var DefaultPathConfig2 = CuePathConfig{ SpecPath: "spec.validation.openAPIV3Schema", } +var OpenAPISpecPathConfig = CuePathConfig{ + NamePath: `x-kubernetes-group-version-kind"[0].kind`, + IdentifierPath: "spec.names.kind", + VersionPath: `"x-kubernetes-group-version-kind"[0].version`, + GroupPath: `"x-kubernetes-group-version-kind"[0].group`, + ScopePath: "spec.scope", + SpecPath: "spec.versions[0].schema.openAPIV3Schema", + PropertiesPath: "properties", +} + var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2} -func Generate(crd string) (component.ComponentDefinition, error) { +func Generate(resource string) (component.ComponentDefinition, error) { cmp := component.ComponentDefinition{} cmp.SchemaVersion = v1beta1.ComponentSchemaVersion cmp.Metadata = component.ComponentDefinition_Metadata{} - crdCue, err := utils.YamlToCue(crd) + crdCue, err := utils.YamlToCue(resource) if err != nil { return cmp, err } + var schema string for _, cfg := range Configs { schema, err = getSchema(crdCue, cfg) diff --git a/utils/component/openapi_generator.go b/utils/component/openapi_generator.go new file mode 100644 index 00000000..f1acf67d --- /dev/null +++ b/utils/component/openapi_generator.go @@ -0,0 +1,172 @@ +package component + +import ( + "encoding/json" + "fmt" + "strings" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/cuecontext" + cueJson "cuelang.org/go/encoding/json" + "github.com/layer5io/meshkit/generators/models" + "github.com/layer5io/meshkit/utils" + "github.com/layer5io/meshkit/utils/manifests" + + "gopkg.in/yaml.v3" + + "github.com/meshery/schemas/models/v1beta1" + "github.com/meshery/schemas/models/v1beta1/category" + "github.com/meshery/schemas/models/v1beta1/component" + "github.com/meshery/schemas/models/v1beta1/model" +) + +func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.ComponentDefinition, error) { + if resource == "" { + return nil, nil + } + resource, err := getResolvedManifest(resource) + if err != nil { + return nil, err + } + cuectx := cuecontext.New() + cueParsedManExpr, err := cueJson.Extract("", []byte(resource)) + if err != nil { + return nil, err + } + + parsedManifest := cuectx.BuildExpr(cueParsedManExpr) + definitions, err := utils.Lookup(parsedManifest, "components.schemas") + + if err != nil { + return nil, err + } + + fields, err := definitions.Fields() + if err != nil { + fmt.Printf("%v\n", err) + return nil, err + } + components := make([]component.ComponentDefinition, 0) + + for fields.Next() { + fieldVal := fields.Value() + kindCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].kind`) + if err != nil { + continue + } + kind, err := kindCue.String() + kind = strings.ToLower(kind) + if err != nil { + fmt.Printf("%v", err) + continue + } + + crd, err := fieldVal.MarshalJSON() + if err != nil { + fmt.Printf("%v", err) + continue + } + versionCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].version`) + if err != nil { + continue + } + + groupCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].group`) + if err != nil { + continue + } + + apiVersion, _ := versionCue.String() + if g, _ := groupCue.String(); g != "" { + apiVersion = g + "/" + apiVersion + } + modified := make(map[string]interface{}) //Remove the given fields which is either not required by End user (like status) or is prefilled by system (like apiVersion, kind and metadata) + err = json.Unmarshal(crd, &modified) + if err != nil { + fmt.Printf("%v", err) + continue + } + + modifiedProps, err := UpdateProperties(fieldVal, cue.ParsePath("properties.spec"), apiVersion) + if err == nil { + modified = modifiedProps + } + + DeleteFields(modified) + crd, err = json.Marshal(modified) + if err != nil { + fmt.Printf("%v", err) + continue + } + + c := component.ComponentDefinition{ + SchemaVersion: v1beta1.ComponentSchemaVersion, + Version: "v1.0.0", + + Format: component.JSON, + Component: component.Component{ + Kind: kind, + Version: apiVersion, + Schema: string(crd), + }, + // Metadata: compMetadata, + DisplayName: manifests.FormatToReadableString(kind), + Model: model.ModelDefinition{ + SchemaVersion: v1beta1.ModelSchemaVersion, + Version: "v1.0.0", + + Model: model.Model{ + Version: pkg.GetVersion(), + }, + Name: pkg.GetName(), + DisplayName: manifests.FormatToReadableString(pkg.GetName()), + Category: category.CategoryDefinition{ + Name: "Orchestration & Management", + }, + Metadata: &model.ModelDefinition_Metadata{ + AdditionalProperties: map[string]interface{}{ + "source_uri": pkg.GetSourceURL(), + }, + }, + }, + } + + components = append(components, c) + } + return components, nil + +} + +func getResolvedManifest(manifest string) (string, error) { + var m map[string]interface{} + + err := yaml.Unmarshal([]byte(manifest), &m) + if err != nil { + return "", utils.ErrDecodeYaml(err) + } + + byt, err := json.Marshal(m) + if err != nil { + return "", utils.ErrMarshal(err) + } + + cuectx := cuecontext.New() + cueParsedManExpr, err := cueJson.Extract("", byt) + if err != nil { + return "", ErrGetSchema(err) + } + + parsedManifest := cuectx.BuildExpr(cueParsedManExpr) + definitions, err := utils.Lookup(parsedManifest, "components.schemas") + if err != nil { + return "", err + } + resol := manifests.ResolveOpenApiRefs{} + cache := make(map[string][]byte) + resolved, err := resol.ResolveReferences(byt, definitions, cache) + if err != nil { + return "", err + } + manifest = string(resolved) + return manifest, nil +} diff --git a/utils/component/utils.go b/utils/component/utils.go index f63d6068..f04ded45 100644 --- a/utils/component/utils.go +++ b/utils/component/utils.go @@ -7,7 +7,6 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" - "gopkg.in/yaml.v2" ) // Remove the fields which is either not required by end user (like status) or is prefilled by system (like apiVersion, kind and metadata) @@ -81,19 +80,10 @@ func FilterCRDs(manifests [][]byte) ([]string, []error) { var errs []error var filteredManifests []string for _, m := range manifests { - - var crd map[string]interface{} - err := yaml.Unmarshal(m, &crd) - if err != nil { - errs = append(errs, err) - continue - } - - isCrd := kubernetes.IsCRD(crd) - if !isCrd { - continue + isCrd := kubernetes.IsCRD(string(m)) + if isCrd { + filteredManifests = append(filteredManifests, string(m)) } - filteredManifests = append(filteredManifests, string(m)) } return filteredManifests, errs } diff --git a/utils/helm/helm.go b/utils/helm/helm.go index 4af65eed..091e5902 100644 --- a/utils/helm/helm.go +++ b/utils/helm/helm.go @@ -9,6 +9,7 @@ import ( "regexp" "strings" + "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/utils" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" @@ -108,7 +109,13 @@ func writeToFile(w io.Writer, path string) error { if err != nil { return utils.ErrReadFile(err, path) } - _, err = w.Write(data) + + byt, err := encoding.ToYaml(data) + if err != nil { + return utils.ErrWriteFile(err, path) + } + + _, err = w.Write(byt) if err != nil { return utils.ErrWriteFile(err, path) } diff --git a/utils/kubernetes/crd.go b/utils/kubernetes/crd.go index 0525a624..9bf8d991 100644 --- a/utils/kubernetes/crd.go +++ b/utils/kubernetes/crd.go @@ -4,6 +4,7 @@ import ( "context" "github.com/layer5io/meshkit/encoding" + "github.com/layer5io/meshkit/utils" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" ) @@ -53,7 +54,19 @@ func GetGVRForCustomResources(crd *CRDItem) *schema.GroupVersionResource { } } -func IsCRD(manifest map[string]interface{}) bool { - kind, ok := manifest["kind"].(string) - return ok && kind == "CustomResourceDefinition" +func IsCRD(manifest string) bool { + cueValue, err := utils.YamlToCue(manifest) + if err != nil { + return false + } + kind, err := utils.Lookup(cueValue, "kind") + if err != nil { + return false + } + kindStr, err := kind.String() + if err != nil { + return false + } + + return kindStr == "CustomResourceDefinition" } diff --git a/utils/manifests/utils.go b/utils/manifests/utils.go index fb3990f7..2935f1c0 100644 --- a/utils/manifests/utils.go +++ b/utils/manifests/utils.go @@ -9,6 +9,7 @@ import ( "strings" "cuelang.org/go/cue" + "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/models/oam/core/v1alpha1" ) @@ -254,7 +255,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue cache = make(map[string][]byte) } var val map[string]interface{} - err := json.Unmarshal(manifest, &val) + err := encoding.Unmarshal(manifest, &val) if err != nil { return nil, err } @@ -266,7 +267,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue if ro.isInsideJsonSchemaProps && (ref == JsonSchemaPropsRef) { // hack so that the UI doesn't crash val["$ref"] = "string" - marVal, errJson := json.Marshal(val) + marVal, errJson := encoding.Marshal(val) if errJson != nil { return manifest, nil } @@ -282,13 +283,13 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue newval = append(newval, v0) continue } - byt, _ := json.Marshal(v0) + byt, _ := encoding.Marshal(v0) byt, err = ro.ResolveReferences(byt, definitions, cache) if err != nil { return nil, err } var newvalmap map[string]interface{} - _ = json.Unmarshal(byt, &newvalmap) + _ = encoding.Unmarshal(byt, &newvalmap) newval = append(newval, newvalmap) } val[k] = newval @@ -333,7 +334,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue if reflect.ValueOf(v).Kind() == reflect.Map { var marVal []byte var def []byte - marVal, err = json.Marshal(v) + marVal, err = encoding.Marshal(v) if err != nil { return nil, err } @@ -349,7 +350,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue } } } - res, err := json.Marshal(val) + res, err := encoding.Marshal(val) if err != nil { return nil, err } @@ -358,7 +359,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue func replaceRefWithVal(def []byte, val map[string]interface{}, k string) error { var defVal map[string]interface{} - err := json.Unmarshal([]byte(def), &defVal) + err := encoding.Unmarshal([]byte(def), &defVal) if err != nil { return err } From 791faefe1475c29ec6365258bb44c05aa1cebd1f Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Thu, 3 Oct 2024 14:57:39 +0530 Subject: [PATCH 101/112] remove hardcoding and handle error wehen schemas are not present Signed-off-by: MUzairS15 --- utils/component/error.go | 2 ++ utils/component/openapi_generator.go | 19 +++++++------------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/utils/component/error.go b/utils/component/error.go index b70e9a00..a37a07bb 100644 --- a/utils/component/error.go +++ b/utils/component/error.go @@ -9,6 +9,8 @@ const ( ErrUpdateSchemaCode = "meshkit-11158" ) +var ErrNoSchemasFound = errors.New(ErrGetSchemaCode, errors.Alert, []string{"Could not get schema for the given openapi spec"}, []string{"The OpenAPI spec doesn't include \"components.schemas\" path"}, []string{"The spec doesn't have include any schema"}, []string{"Verify the spec has valid schema."}) + // No reference usage found. Also check in adapters before deleting func ErrCrdGenerate(err error) error { return errors.New(ErrCrdGenerateCode, errors.Alert, []string{"Could not generate component with the given CRD"}, []string{err.Error()}, []string{""}, []string{"Verify CRD has valid schema."}) diff --git a/utils/component/openapi_generator.go b/utils/component/openapi_generator.go index f1acf67d..e7031d55 100644 --- a/utils/component/openapi_generator.go +++ b/utils/component/openapi_generator.go @@ -2,6 +2,7 @@ package component import ( "encoding/json" + "errors" "fmt" "strings" @@ -15,7 +16,6 @@ import ( "gopkg.in/yaml.v3" "github.com/meshery/schemas/models/v1beta1" - "github.com/meshery/schemas/models/v1beta1/category" "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/model" ) @@ -25,6 +25,9 @@ func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.Compo return nil, nil } resource, err := getResolvedManifest(resource) + if err != nil && errors.Is(err, ErrNoSchemasFound) { + return nil, nil + } if err != nil { return nil, err } @@ -36,7 +39,7 @@ func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.Compo parsedManifest := cuectx.BuildExpr(cueParsedManExpr) definitions, err := utils.Lookup(parsedManifest, "components.schemas") - + if err != nil { return nil, err } @@ -101,28 +104,20 @@ func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.Compo c := component.ComponentDefinition{ SchemaVersion: v1beta1.ComponentSchemaVersion, - Version: "v1.0.0", - - Format: component.JSON, + Format: component.JSON, Component: component.Component{ Kind: kind, Version: apiVersion, Schema: string(crd), }, - // Metadata: compMetadata, DisplayName: manifests.FormatToReadableString(kind), Model: model.ModelDefinition{ SchemaVersion: v1beta1.ModelSchemaVersion, - Version: "v1.0.0", - Model: model.Model{ Version: pkg.GetVersion(), }, Name: pkg.GetName(), DisplayName: manifests.FormatToReadableString(pkg.GetName()), - Category: category.CategoryDefinition{ - Name: "Orchestration & Management", - }, Metadata: &model.ModelDefinition_Metadata{ AdditionalProperties: map[string]interface{}{ "source_uri": pkg.GetSourceURL(), @@ -159,7 +154,7 @@ func getResolvedManifest(manifest string) (string, error) { parsedManifest := cuectx.BuildExpr(cueParsedManExpr) definitions, err := utils.Lookup(parsedManifest, "components.schemas") if err != nil { - return "", err + return "", ErrNoSchemasFound } resol := manifests.ResolveOpenApiRefs{} cache := make(map[string][]byte) From da74630b9f3aa798b25d1ab8a00169dd38cd3754 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Thu, 3 Oct 2024 22:08:41 +0530 Subject: [PATCH 102/112] remove processing for 'kind' Signed-off-by: MUzairS15 --- utils/component/openapi_generator.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/utils/component/openapi_generator.go b/utils/component/openapi_generator.go index e7031d55..045fcd6f 100644 --- a/utils/component/openapi_generator.go +++ b/utils/component/openapi_generator.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "strings" "cuelang.org/go/cue" "cuelang.org/go/cue/cuecontext" @@ -39,7 +38,7 @@ func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.Compo parsedManifest := cuectx.BuildExpr(cueParsedManExpr) definitions, err := utils.Lookup(parsedManifest, "components.schemas") - + if err != nil { return nil, err } @@ -58,7 +57,6 @@ func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.Compo continue } kind, err := kindCue.String() - kind = strings.ToLower(kind) if err != nil { fmt.Printf("%v", err) continue From b4716964441f4b66eaaa8fad564d19190fdba4dc Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Fri, 4 Oct 2024 22:43:30 +0530 Subject: [PATCH 103/112] check and bypass on basis of status w/signoff Signed-off-by: Jougan-0 --- models/registration/register.go | 11 ++++++++++- utils/utils.go | 4 +++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/models/registration/register.go b/models/registration/register.go index 02dde7f5..14005b7f 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -51,9 +51,13 @@ func (rh *RegistrationHelper) register(pkg PackagingUnit) { //silently exit if the model does not conatin any components or relationships return } + ignored := model.ModelDefinitionStatusIgnored // 1. Register the model model := pkg.Model - + modelstatus := model.Status + if modelstatus == ignored { + return + } // Don't register anything else if registrant is not there if model.Registrant.Kind == "" { err := ErrMissingRegistrant(model.Name) @@ -100,6 +104,11 @@ func (rh *RegistrationHelper) register(pkg PackagingUnit) { var registeredRelationships []relationship.RelationshipDefinition // 2. Register components for _, comp := range pkg.Components { + status := *comp.Status + if status == component.Ignored { + continue + } + comp.Model = model if comp.Styles != nil { diff --git a/utils/utils.go b/utils/utils.go index 737ca1c0..25a7338a 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -402,7 +402,9 @@ func CreateDirectory(path string) error { func ReplaceSpacesAndConvertToLowercase(s string) string { return strings.ToLower(strings.ReplaceAll(s, " ", "")) } - +func ReplaceSpacesWithHyphenAndConvertToLowercase(s string) string { + return strings.ToLower(strings.ReplaceAll(s, " ", "-")) +} func ExtractDomainFromURL(location string) string { parsedURL, err := url.Parse(location) // If unable to extract domain return the location as is. From d7995640e723bdb718c2033d05ad5ef6af6af529 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Tue, 8 Oct 2024 23:08:14 +0530 Subject: [PATCH 104/112] assign missing namespace w/signoff Signed-off-by: Jougan-0 --- utils/component/openapi_generator.go | 44 ++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/utils/component/openapi_generator.go b/utils/component/openapi_generator.go index 045fcd6f..8d0178de 100644 --- a/utils/component/openapi_generator.go +++ b/utils/component/openapi_generator.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "cuelang.org/go/cue" "cuelang.org/go/cue/cuecontext" @@ -100,6 +101,26 @@ func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.Compo continue } + // Determine if the resource is namespaced + var isNamespaced bool + + scopeCue, err := utils.Lookup(fieldVal, `"x-kubernetes-resource".scope`) + if err == nil { + scope, err := scopeCue.String() + if err == nil { + if scope == "Namespaced" { + isNamespaced = true + } else if scope == "Cluster" { + isNamespaced = false + } + } + } else { + isNamespaced, err = getResourceScope(resource, kind) + if err != nil { + isNamespaced = false + } + } + c := component.ComponentDefinition{ SchemaVersion: v1beta1.ComponentSchemaVersion, Format: component.JSON, @@ -109,6 +130,9 @@ func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.Compo Schema: string(crd), }, DisplayName: manifests.FormatToReadableString(kind), + Metadata: component.ComponentDefinition_Metadata{ + IsNamespaced: isNamespaced, + }, Model: model.ModelDefinition{ SchemaVersion: v1beta1.ModelSchemaVersion, Model: model.Model{ @@ -127,7 +151,27 @@ func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.Compo components = append(components, c) } return components, nil +} +func getResourceScope(manifest string, kind string) (bool, error) { + var m map[string]interface{} + + err := yaml.Unmarshal([]byte(manifest), &m) + if err != nil { + return false, utils.ErrDecodeYaml(err) + } + + paths, ok := m["paths"].(map[string]interface{}) + if !ok { + return false, fmt.Errorf("paths not found in manifest") + } + + for path := range paths { + if strings.Contains(path, "/namespaces/{namespace}/") && strings.Contains(path, strings.ToLower(kind)) { + return true, nil // Resource is namespaced + } + } + return false, nil // Resource is cluster-scoped } func getResolvedManifest(manifest string) (string, error) { From 35ea71ee1b6a90a7a89ff9cbab2a0b0c49e8ba35 Mon Sep 17 00:00:00 2001 From: Yash Sharma Date: Sun, 13 Oct 2024 11:58:27 +0530 Subject: [PATCH 105/112] Added missing condition for id filter for entities Signed-off-by: Yash Sharma --- models/meshmodel/registry/v1alpha3/relationship_filter.go | 3 +++ models/meshmodel/registry/v1beta1/category_filter.go | 3 +++ models/meshmodel/registry/v1beta1/component_filter.go | 4 +++- models/meshmodel/registry/v1beta1/model_filter.go | 3 +++ models/meshmodel/registry/v1beta1/policy_filter.go | 4 +++- 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/models/meshmodel/registry/v1alpha3/relationship_filter.go b/models/meshmodel/registry/v1alpha3/relationship_filter.go index 624c683a..de31351e 100644 --- a/models/meshmodel/registry/v1alpha3/relationship_filter.go +++ b/models/meshmodel/registry/v1alpha3/relationship_filter.go @@ -74,6 +74,9 @@ func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entit if relationshipFilter.SubType != "" { finder = finder.Where("relationship_definition_dbs.sub_type = ?", relationshipFilter.SubType) } + if relationshipFilter.Id != "" { + finder = finder.Where("relationship_definition_dbs.id = ?", relationshipFilter.Id) + } if relationshipFilter.ModelName != "" { finder = finder.Where("model_dbs.name = ?", relationshipFilter.ModelName) } diff --git a/models/meshmodel/registry/v1beta1/category_filter.go b/models/meshmodel/registry/v1beta1/category_filter.go index 399385a1..520614ae 100644 --- a/models/meshmodel/registry/v1beta1/category_filter.go +++ b/models/meshmodel/registry/v1beta1/category_filter.go @@ -50,6 +50,9 @@ func (cf *CategoryFilter) Get(db *database.Handler) ([]entity.Entity, int64, int finder = finder.Where("name = ?", cf.Name) } } + if cf.Id != "" { + finder = finder.Where("id = ?", cf.Id) + } if cf.OrderOn != "" { if cf.Sort == "desc" { finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: cf.OrderOn}, Desc: true}) diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index 76a6fb57..47aefe58 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -116,7 +116,9 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti if componentFilter.Version != "" { finder = finder.Where("model_dbs.model->>'version' = ?", componentFilter.Version) } - + if componentFilter.Id != "" { + finder = finder.Where("component_definition_dbs.id = ?", componentFilter.Id) + } if componentFilter.OrderOn != "" { if componentFilter.Sort == "desc" { finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: componentFilter.OrderOn}, Desc: true}) diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index 6a512f79..c52e3d27 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -138,6 +138,9 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e } else if mf.Annotations == "false" { finder = finder.Where("model_dbs.metadata->>'isAnnotation' = false") } + if mf.Id != "" { + finder = finder.Where("model_dbs.id = ?", mf.Id) + } if mf.OrderOn != "" { if mf.Sort == "desc" { finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: mf.OrderOn}, Desc: true}) diff --git a/models/meshmodel/registry/v1beta1/policy_filter.go b/models/meshmodel/registry/v1beta1/policy_filter.go index d15d970c..3108d68c 100644 --- a/models/meshmodel/registry/v1beta1/policy_filter.go +++ b/models/meshmodel/registry/v1beta1/policy_filter.go @@ -49,7 +49,9 @@ func (pf *PolicyFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, if pf.ModelName != "" { finder = finder.Where("model_dbs.name = ?", pf.ModelName) } - + if pf.Id != "" { + finder = finder.Where("policy_definition_dbs.id = ?", pf.Id) + } var count int64 finder.Count(&count) From 62137a0f8fbcd6b3e43c4df9c653bf079ae6cd99 Mon Sep 17 00:00:00 2001 From: Lee Calcote Date: Sun, 13 Oct 2024 19:29:57 -0500 Subject: [PATCH 106/112] Update CONTRIBUTING.md Signed-off-by: Lee Calcote --- CONTRIBUTING.md | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 00417cc3..d254b26e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,22 +50,6 @@ Or you may configure your IDE, for example, Visual Studio Code to automatically -## Documentation Contribution Flow -Please contribute! Layer5 documentation uses Jekyll and GitHub Pages to host docs sites. Learn more about [Layer5's documentation framework](https://docs.google.com/document/d/17guuaxb0xsfutBCzyj2CT6OZiFnMu9w4PzoILXhRXSo/edit?usp=sharing). The process of contributing follows this flow: - -1. Create a fork, if you have not already, by following the steps described [here](./CONTRIBUTING-gitflow.md) -1. In the local copy of your fork, navigate to the docs folder. -`cd docs` -1. Create and checkout a new branch to make changes within -`git checkout -b ` -1. Edit/add documentation. -`vi .md` -1. Run site locally to preview changes. -`make site` -1. Commit, [sign-off](#commit-signing), and push changes to your remote branch. -`git push origin ` -1. Open a pull request (in your web browser) against the repo. - #### Tests Users can now test their code on their local machine against the CI checks implemented using `make run-tests`. @@ -92,7 +76,7 @@ All contributors are invited to review pull requests. See this short video on [h Resources: https://lab.github.com and https://try.github.com/ -# FQA +# FAQs ### Instructions for making custom dictionary entries From 987c093aee82a0501552e438ffc1f059cf534a77 Mon Sep 17 00:00:00 2001 From: Yash Sharma Date: Mon, 14 Oct 2024 16:06:15 +0530 Subject: [PATCH 107/112] Update meshkit for trim filter in model Signed-off-by: Yash Sharma --- .../registry/v1beta1/model_filter.go | 60 +++++++++++++------ 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index c52e3d27..cb014d5b 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -29,6 +29,9 @@ type ModelFilter struct { Components bool Relationships bool Status string + // When Trim is true it will only send necessary models data + // like: component count, relationship count, id and name of model + Trim bool } // Create the filter from map[string]interface{} @@ -182,27 +185,48 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e for _, modelDB := range modelWithCategories { // resolve for loop scope _modelDB := modelDB - if includeComponents { - var components []component.ComponentDefinition - finder := db.Model(&component.ComponentDefinition{}). - Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version,component_definition_dbs.styles,component_definition_dbs.capabilities"). - Where("component_definition_dbs.model_id = ?", _modelDB.Id) - if err := finder.Scan(&components).Error; err != nil { - return nil, 0, 0, err + var componentCount int64 + db.Model(&component.ComponentDefinition{}).Where("component_definition_dbs.model_id = ?", _modelDB.Id).Count(&componentCount) + var relationshipCount int64 + db.Model(&relationship.RelationshipDefinition{}).Where("relationship_definition_dbs.model_id = ?", _modelDB.Id).Count(&relationshipCount) + _modelDB.ComponentsCount = int(componentCount) + _modelDB.RelationshipsCount = int(relationshipCount) + + // If Trim is true, only include the id, name, counts and metadata + if mf.Trim { + trimmedModel := &model.ModelDefinition{ + Id: _modelDB.Id, + Name: _modelDB.Name, + DisplayName: _modelDB.DisplayName, + Metadata: _modelDB.Metadata, + ComponentsCount: int(componentCount), + RelationshipsCount: int(relationshipCount), } - _modelDB.Components = components - } - if includeRelationships { - var relationships []relationship.RelationshipDefinition - finder := db.Model(&relationship.RelationshipDefinition{}). - Select("relationship_definition_dbs.*"). - Where("relationship_definition_dbs.model_id = ?", _modelDB.Id) - if err := finder.Scan(&relationships).Error; err != nil { - return nil, 0, 0, err + defs = append(defs, trimmedModel) + + } else { + if includeComponents { + var components []component.ComponentDefinition + finder := db.Model(&component.ComponentDefinition{}). + Select("component_definition_dbs.*"). + Where("component_definition_dbs.model_id = ?", _modelDB.Id) + if err := finder.Scan(&components).Error; err != nil { + return nil, 0, 0, err + } + _modelDB.Components = components + } + if includeRelationships { + var relationships []relationship.RelationshipDefinition + finder := db.Model(&relationship.RelationshipDefinition{}). + Select("relationship_definition_dbs.*"). + Where("relationship_definition_dbs.model_id = ?", _modelDB.Id) + if err := finder.Scan(&relationships).Error; err != nil { + return nil, 0, 0, err + } + _modelDB.Relationships = relationships } - _modelDB.Relationships = relationships + defs = append(defs, &_modelDB) } - defs = append(defs, &_modelDB) } return defs, count, countUniqueModels(modelWithCategories), nil } From 21e7b725354ee7239c7f6f8c92b7a85be83e855f Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Mon, 14 Oct 2024 18:15:59 +0530 Subject: [PATCH 108/112] add missing namespace w/singoff Signed-off-by: Jougan-0 --- converter/k8s.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/converter/k8s.go b/converter/k8s.go index 10c776ca..f9ad65ab 100644 --- a/converter/k8s.go +++ b/converter/k8s.go @@ -39,6 +39,7 @@ func NewK8sManifestsFromPatternfile(patternFile *pattern.PatternFile) (string, e func CreateK8sResourceStructure(comp *component.ComponentDefinition) map[string]interface{} { annotations := map[string]interface{}{} labels := map[string]interface{}{} + namespace := "default" _confMetadata, ok := comp.Configuration["metadata"] if ok { @@ -51,10 +52,14 @@ func CreateK8sResourceStructure(comp *component.ComponentDefinition) map[string] } _label, ok := confMetadata["labels"] - if ok { labels, _ = utils.Cast[map[string]interface{}](_label) } + + _ns, ok := confMetadata["namespace"] + if ok { + namespace, _ = utils.Cast[string](_ns) + } } } @@ -65,11 +70,12 @@ func CreateK8sResourceStructure(comp *component.ComponentDefinition) map[string] "name": comp.DisplayName, "annotations": annotations, "labels": labels, + "namespace": namespace, }, } for k, v := range comp.Configuration { - if k == "apiVersion" || k == "kind" || k == "metadata" { + if k == "apiVersion" || k == "kind" || k == "metadata" || k == "namespace" { continue } From f0f3ec261833ef67363105549acf68ea787dbd96 Mon Sep 17 00:00:00 2001 From: Lee Calcote Date: Mon, 14 Oct 2024 23:17:05 -0500 Subject: [PATCH 109/112] Update slack.yml Signed-off-by: Lee Calcote --- .github/workflows/slack.yml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/slack.yml b/.github/workflows/slack.yml index afd9a5b3..b7b85f31 100644 --- a/.github/workflows/slack.yml +++ b/.github/workflows/slack.yml @@ -1,16 +1,30 @@ -name: Slack Notify on Star -on: watch +name: Slack Notify +on: + watch: + types: [started] jobs: star-notify: + if: github.event_name == 'watch' name: Notify Slack on star - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Get current star count run: | - echo "STARS=$(curl --silent 'https://api.github.com/repos/layer5io/meshkit' -H 'Accept: application/vnd.github.preview' | jq '.watchers_count')" >> $GITHUB_ENV + echo "STARS=$(curl --silent 'https://api.github.com/repos/${{github.repository}}' -H 'Accept: application/vnd.github.preview' | jq '.stargazers_count')" >> $GITHUB_ENV - name: Notify slack env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} uses: pullreminders/slack-action@master with: - args: '{\"channel\":\"CSK7N9TGX\",\"text\":\"${{ github.actor }} just starred Meshkit! (https://github.com/layer5io/meshkit/stargazers) Total ⭐️: ${{env.STARS}}\"}' + args: '{\"channel\":\"CSK7N9TGX\",\"text\":\"${{ github.actor }} just starred ${{github.repository}}! (https://github.com/${{github.repository}}/stargazers) Total ⭐️: ${{env.STARS}}\"}' + good-first-issue-notify: + if: github.event_name == 'issues' && github.event.label.name == 'good first issue' || github.event.label.name == 'first-timers-only' + name: Notify Slack for new good-first-issue + runs-on: ubuntu-22.04 + steps: + - name: Notify slack + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + uses: pullreminders/slack-action@master + with: + args: '{\"channel\":\"C019426UBNY\",\"text\":\"A good first issue label was just added to ${{github.event.issue.html_url}}.\"}' From f45ae457686e2bfe505530374340a2e481026944 Mon Sep 17 00:00:00 2001 From: Yash Sharma Date: Tue, 15 Oct 2024 12:01:59 +0530 Subject: [PATCH 110/112] Bump schemas repo Signed-off-by: Yash Sharma --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index ea10a40a..8e252044 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/google/uuid v1.6.0 github.com/kubernetes/kompose v1.31.1 github.com/layer5io/meshery-operator v0.7.0 - github.com/meshery/schemas v0.7.31 + github.com/meshery/schemas v0.7.36 github.com/nats-io/nats.go v1.31.0 github.com/open-policy-agent/opa v0.67.1 github.com/opencontainers/image-spec v1.1.0 @@ -39,7 +39,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/postgres v1.5.3 gorm.io/driver/sqlite v1.5.4 - gorm.io/gorm v1.25.11 + gorm.io/gorm v1.25.12 helm.sh/helm/v3 v3.13.2 k8s.io/api v0.28.4 k8s.io/apimachinery v0.28.4 diff --git a/go.sum b/go.sum index 9aad5ca9..f4b94cd3 100644 --- a/go.sum +++ b/go.sum @@ -608,8 +608,8 @@ github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/meshery/kompose v1.0.1 h1:lg8B/pkLh6762jeFsQATD8UJZZwXZf/aviC3/dzw78A= github.com/meshery/kompose v1.0.1/go.mod h1:TWhWTEMbJBUzENf4JTEtBmZRFm/r8n0nS6v4/nSD2vA= -github.com/meshery/schemas v0.7.31 h1:FfVR+oErAiiEomt6sTZI5uKhoyU26QXawT6UDZHbthI= -github.com/meshery/schemas v0.7.31/go.mod h1:UfiO+zm92yLkaJP0aroNwVnjuozoh793AWDXrKDYmT0= +github.com/meshery/schemas v0.7.36 h1:KQOz/SODr+T6fDS2lDSrNo/Eu3LCaEUGIRauQtvXkjg= +github.com/meshery/schemas v0.7.36/go.mod h1:wOh519/EDxiYlC4aeGv74ru+t9h9VJ4P2JYIvSfdPWQ= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= @@ -1198,8 +1198,8 @@ gorm.io/driver/postgres v1.5.3 h1:qKGY5CPHOuj47K/VxbCXJfFvIUeqMSXXadqdCY+MbBU= gorm.io/driver/postgres v1.5.3/go.mod h1:F+LtvlFhZT7UBiA81mC9W6Su3D4WUhSboc/36QZU0gk= gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0= gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= -gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= -gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= +gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= From a9284c710a0fa6d9d69c29a2c48df1f3e6944b97 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Tue, 15 Oct 2024 18:55:29 +0530 Subject: [PATCH 111/112] remove deduplication for models w/signoff Signed-off-by: Jougan-0 --- models/registration/register.go | 2 ++ models/registration/svg_helper.go | 10 ++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/models/registration/register.go b/models/registration/register.go index 14005b7f..d93661bc 100644 --- a/models/registration/register.go +++ b/models/registration/register.go @@ -81,6 +81,7 @@ func (rh *RegistrationHelper) register(pkg PackagingUnit) { rh.svgBaseDir, model.Name, model.Name, + true, ) if svgCompletePath != "" { model.Metadata.SvgComplete = &svgCompletePath @@ -120,6 +121,7 @@ func (rh *RegistrationHelper) register(pkg PackagingUnit) { rh.svgBaseDir, comp.Model.Name, comp.Component.Kind, + false, ) } diff --git a/models/registration/svg_helper.go b/models/registration/svg_helper.go index 5122c795..05b77142 100644 --- a/models/registration/svg_helper.go +++ b/models/registration/svg_helper.go @@ -20,7 +20,7 @@ func writeHashCheckSVG(key string, val string) { mx.Unlock() } -func WriteAndReplaceSVGWithFileSystemPath(svgColor, svgWhite, svgComplete string, baseDir, dirname, filename string) (svgColorPath, svgWhitePath, svgCompletePath string) { +func WriteAndReplaceSVGWithFileSystemPath(svgColor, svgWhite, svgComplete string, baseDir, dirname, filename string, isModel bool) (svgColorPath, svgWhitePath, svgCompletePath string) { filename = strings.ToLower(filename) successCreatingDirectory := false defer func() { @@ -40,7 +40,7 @@ func WriteAndReplaceSVGWithFileSystemPath(svgColor, svgWhite, svgComplete string hash := md5.Sum([]byte(svgColor)) hashString := hex.EncodeToString(hash[:]) pathsvg := hashCheckSVG[hashString] - if pathsvg != "" { // the image has already been loaded, point the component to that path + if pathsvg != "" && !isModel { // the image has already been loaded, point the component to that path svgColorPath = pathsvg goto White } @@ -59,7 +59,7 @@ func WriteAndReplaceSVGWithFileSystemPath(svgColor, svgWhite, svgComplete string } White: - if svgWhite != "" { + if svgWhite != "" && !isModel { path := filepath.Join(baseDir, dirname, "white") err := os.MkdirAll(path, 0777) if err != nil { @@ -90,7 +90,7 @@ White: } Complete: - if svgComplete != "" { + if svgComplete != "" && !isModel { path := filepath.Join(baseDir, dirname, "complete") err := os.MkdirAll(path, 0777) if err != nil { @@ -123,8 +123,6 @@ Complete: return } -// func WriteAndReplaceSVGWithFileSystemPath(metadata map[string]interface{}, baseDir, dirname, filename string) { - func getRelativePathForAPI(baseDir, path string) string { ui := strings.TrimPrefix(baseDir, "../../") return filepath.Join(ui, path) From 35d1cbf52ade71448a7a71084b6cdf580b32ce74 Mon Sep 17 00:00:00 2001 From: Jougan-0 Date: Thu, 17 Oct 2024 20:56:16 +0530 Subject: [PATCH 112/112] update model logic w/signoff Signed-off-by: Jougan-0 --- models/registration/svg_helper.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/models/registration/svg_helper.go b/models/registration/svg_helper.go index 05b77142..351d22ee 100644 --- a/models/registration/svg_helper.go +++ b/models/registration/svg_helper.go @@ -59,7 +59,7 @@ func WriteAndReplaceSVGWithFileSystemPath(svgColor, svgWhite, svgComplete string } White: - if svgWhite != "" && !isModel { + if svgWhite != "" { path := filepath.Join(baseDir, dirname, "white") err := os.MkdirAll(path, 0777) if err != nil { @@ -71,7 +71,7 @@ White: hash := md5.Sum([]byte(svgWhite)) hashString := hex.EncodeToString(hash[:]) pathsvg := hashCheckSVG[hashString] - if pathsvg != "" { // the image has already been loaded, point the component to that path + if pathsvg != "" && !isModel { // the image has already been loaded, point the component to that path svgWhitePath = pathsvg goto Complete } @@ -90,7 +90,7 @@ White: } Complete: - if svgComplete != "" && !isModel { + if svgComplete != "" { path := filepath.Join(baseDir, dirname, "complete") err := os.MkdirAll(path, 0777) if err != nil { @@ -102,7 +102,7 @@ Complete: hash := md5.Sum([]byte(svgComplete)) hashString := hex.EncodeToString(hash[:]) pathsvg := hashCheckSVG[hashString] - if pathsvg != "" { // the image has already been loaded, point the component to that path + if pathsvg != "" && !isModel { // the image has already been loaded, point the component to that path svgCompletePath = pathsvg return }