From 12ec8f66b1b1a3b47628a56b181112b0b648d5d1 Mon Sep 17 00:00:00 2001 From: aabidsofi19 Date: Sun, 29 Sep 2024 22:23:41 +0530 Subject: [PATCH 1/2] Update import design schema Signed-off-by: aabidsofi19 --- schemas/configuration/designImport.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/schemas/configuration/designImport.json b/schemas/configuration/designImport.json index 4c5752af..f59b205b 100644 --- a/schemas/configuration/designImport.json +++ b/schemas/configuration/designImport.json @@ -3,7 +3,7 @@ "properties": { "name": { "type": "string", - "title": "Design file name", + "title": "Design filename", "default": "Untitled Design", "x-rjsf-grid-area": "6", "description": "Provide a name for your design file. This name will help you identify the file more easily. You can also change the name of your design after importing it." @@ -16,7 +16,6 @@ "Docker Compose", "Meshery Design" ], - "default": "Meshery Design", "x-rjsf-grid-area": "6", "description": "Select the type of design you are uploading. The 'Design Type' determines the format, structure, and content of the file you are uploading. Choose the appropriate design type that matches the nature of your file. Checkout https://docs.meshery.io/guides/meshery-design to learn more about designs" } @@ -126,4 +125,4 @@ ] } ] -} \ No newline at end of file +} From d889a68359a1f9de0462a53e06ca21c9aca62cc1 Mon Sep 17 00:00:00 2001 From: Lee Calcote Date: Mon, 30 Sep 2024 16:24:54 -0500 Subject: [PATCH 2/2] Revert "support generation of model from openapi schemas" --- encoding/convert.go | 15 --- generators/artifacthub/package.go | 8 -- generators/github/package.go | 59 +++------ generators/models/interfaces.go | 2 - utils/component/generator.go | 15 +-- utils/component/openapi_generator.go | 172 --------------------------- utils/component/utils.go | 16 ++- utils/helm/helm.go | 9 +- utils/kubernetes/crd.go | 19 +-- utils/manifests/utils.go | 15 ++- 10 files changed, 45 insertions(+), 285 deletions(-) delete mode 100644 encoding/convert.go delete mode 100644 utils/component/openapi_generator.go diff --git a/encoding/convert.go b/encoding/convert.go deleted file mode 100644 index 5dc810eb..00000000 --- a/encoding/convert.go +++ /dev/null @@ -1,15 +0,0 @@ -package encoding - -import ( - "gopkg.in/yaml.v3" -) - -func ToYaml(data []byte) ([]byte, error) { - var out map[string]interface{} - err := Unmarshal(data, &out) - if err != nil { - return nil, err - } - - return yaml.Marshal(out) -} diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index e40509d8..61b3d8be 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -38,14 +38,6 @@ func (pkg AhPackage) GetVersion() string { return pkg.Version } -func (pkg AhPackage) GetSourceURL() string { - return pkg.ChartUrl -} - -func (pkg AhPackage) GetName() string { - return pkg.Name -} - func (pkg AhPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { components := make([]_component.ComponentDefinition, 0) // TODO: Move this to the configuration diff --git a/generators/github/package.go b/generators/github/package.go index e7efea6c..0c632ef9 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -6,7 +6,6 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" - "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1/category" _component "github.com/meshery/schemas/models/v1beta1/component" @@ -26,14 +25,6 @@ func (gp GitHubPackage) GetVersion() string { return gp.version } -func (gp GitHubPackage) GetSourceURL() string { - return gp.SourceURL -} - -func (gp GitHubPackage) GetName() string { - return gp.Name -} - func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, error) { components := make([]_component.ComponentDefinition, 0) @@ -43,40 +34,28 @@ func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, } manifestBytes := bytes.Split(data, []byte("\n---\n")) - errs := []error{} - - for _, crd := range manifestBytes { - isCrd := kubernetes.IsCRD(string(crd)) - if !isCrd { + crds, errs := component.FilterCRDs(manifestBytes) - comps, err := component.GenerateFromOpenAPI(string(crd), gp) - if err != nil { - errs = append(errs, component.ErrGetSchema(err)) - continue - } - components = append(components, comps...) - } else { - comp, err := component.Generate(string(crd)) - if err != nil { - continue - } - if comp.Model.Metadata == nil { - comp.Model.Metadata = &model.ModelDefinition_Metadata{} - } - if comp.Model.Metadata.AdditionalProperties == nil { - comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) - } - - comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL - comp.Model.Version = gp.version - comp.Model.Name = gp.Name - comp.Model.Category = category.CategoryDefinition{ - Name: "", - } - comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) - components = append(components, comp) + for _, crd := range crds { + comp, err := component.Generate(crd) + if err != nil { + continue + } + if comp.Model.Metadata == nil { + comp.Model.Metadata = &model.ModelDefinition_Metadata{} + } + if comp.Model.Metadata.AdditionalProperties == nil { + comp.Model.Metadata.AdditionalProperties = make(map[string]interface{}) } + comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL + comp.Model.Version = gp.version + comp.Model.Name = gp.Name + comp.Model.Category = category.CategoryDefinition{ + Name: "", + } + comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) + components = append(components, comp) } return components, utils.CombineErrors(errs, "\n") diff --git a/generators/models/interfaces.go b/generators/models/interfaces.go index c6dcd3eb..19f61177 100644 --- a/generators/models/interfaces.go +++ b/generators/models/interfaces.go @@ -13,8 +13,6 @@ type Validator interface { type Package interface { GenerateComponents() ([]component.ComponentDefinition, error) GetVersion() string - GetSourceURL() string - GetName() string } // Supports pulling packages from Artifact Hub and other sources like Docker Hub. diff --git a/utils/component/generator.go b/utils/component/generator.go index b228b363..288f2780 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -45,28 +45,17 @@ var DefaultPathConfig2 = CuePathConfig{ SpecPath: "spec.validation.openAPIV3Schema", } -var OpenAPISpecPathConfig = CuePathConfig{ - NamePath: `x-kubernetes-group-version-kind"[0].kind`, - IdentifierPath: "spec.names.kind", - VersionPath: `"x-kubernetes-group-version-kind"[0].version`, - GroupPath: `"x-kubernetes-group-version-kind"[0].group`, - ScopePath: "spec.scope", - SpecPath: "spec.versions[0].schema.openAPIV3Schema", - PropertiesPath: "properties", -} - var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2} -func Generate(resource string) (component.ComponentDefinition, error) { +func Generate(crd string) (component.ComponentDefinition, error) { cmp := component.ComponentDefinition{} cmp.SchemaVersion = v1beta1.ComponentSchemaVersion cmp.Metadata = component.ComponentDefinition_Metadata{} - crdCue, err := utils.YamlToCue(resource) + crdCue, err := utils.YamlToCue(crd) if err != nil { return cmp, err } - var schema string for _, cfg := range Configs { schema, err = getSchema(crdCue, cfg) diff --git a/utils/component/openapi_generator.go b/utils/component/openapi_generator.go deleted file mode 100644 index f1acf67d..00000000 --- a/utils/component/openapi_generator.go +++ /dev/null @@ -1,172 +0,0 @@ -package component - -import ( - "encoding/json" - "fmt" - "strings" - - "cuelang.org/go/cue" - "cuelang.org/go/cue/cuecontext" - cueJson "cuelang.org/go/encoding/json" - "github.com/layer5io/meshkit/generators/models" - "github.com/layer5io/meshkit/utils" - "github.com/layer5io/meshkit/utils/manifests" - - "gopkg.in/yaml.v3" - - "github.com/meshery/schemas/models/v1beta1" - "github.com/meshery/schemas/models/v1beta1/category" - "github.com/meshery/schemas/models/v1beta1/component" - "github.com/meshery/schemas/models/v1beta1/model" -) - -func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.ComponentDefinition, error) { - if resource == "" { - return nil, nil - } - resource, err := getResolvedManifest(resource) - if err != nil { - return nil, err - } - cuectx := cuecontext.New() - cueParsedManExpr, err := cueJson.Extract("", []byte(resource)) - if err != nil { - return nil, err - } - - parsedManifest := cuectx.BuildExpr(cueParsedManExpr) - definitions, err := utils.Lookup(parsedManifest, "components.schemas") - - if err != nil { - return nil, err - } - - fields, err := definitions.Fields() - if err != nil { - fmt.Printf("%v\n", err) - return nil, err - } - components := make([]component.ComponentDefinition, 0) - - for fields.Next() { - fieldVal := fields.Value() - kindCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].kind`) - if err != nil { - continue - } - kind, err := kindCue.String() - kind = strings.ToLower(kind) - if err != nil { - fmt.Printf("%v", err) - continue - } - - crd, err := fieldVal.MarshalJSON() - if err != nil { - fmt.Printf("%v", err) - continue - } - versionCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].version`) - if err != nil { - continue - } - - groupCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].group`) - if err != nil { - continue - } - - apiVersion, _ := versionCue.String() - if g, _ := groupCue.String(); g != "" { - apiVersion = g + "/" + apiVersion - } - modified := make(map[string]interface{}) //Remove the given fields which is either not required by End user (like status) or is prefilled by system (like apiVersion, kind and metadata) - err = json.Unmarshal(crd, &modified) - if err != nil { - fmt.Printf("%v", err) - continue - } - - modifiedProps, err := UpdateProperties(fieldVal, cue.ParsePath("properties.spec"), apiVersion) - if err == nil { - modified = modifiedProps - } - - DeleteFields(modified) - crd, err = json.Marshal(modified) - if err != nil { - fmt.Printf("%v", err) - continue - } - - c := component.ComponentDefinition{ - SchemaVersion: v1beta1.ComponentSchemaVersion, - Version: "v1.0.0", - - Format: component.JSON, - Component: component.Component{ - Kind: kind, - Version: apiVersion, - Schema: string(crd), - }, - // Metadata: compMetadata, - DisplayName: manifests.FormatToReadableString(kind), - Model: model.ModelDefinition{ - SchemaVersion: v1beta1.ModelSchemaVersion, - Version: "v1.0.0", - - Model: model.Model{ - Version: pkg.GetVersion(), - }, - Name: pkg.GetName(), - DisplayName: manifests.FormatToReadableString(pkg.GetName()), - Category: category.CategoryDefinition{ - Name: "Orchestration & Management", - }, - Metadata: &model.ModelDefinition_Metadata{ - AdditionalProperties: map[string]interface{}{ - "source_uri": pkg.GetSourceURL(), - }, - }, - }, - } - - components = append(components, c) - } - return components, nil - -} - -func getResolvedManifest(manifest string) (string, error) { - var m map[string]interface{} - - err := yaml.Unmarshal([]byte(manifest), &m) - if err != nil { - return "", utils.ErrDecodeYaml(err) - } - - byt, err := json.Marshal(m) - if err != nil { - return "", utils.ErrMarshal(err) - } - - cuectx := cuecontext.New() - cueParsedManExpr, err := cueJson.Extract("", byt) - if err != nil { - return "", ErrGetSchema(err) - } - - parsedManifest := cuectx.BuildExpr(cueParsedManExpr) - definitions, err := utils.Lookup(parsedManifest, "components.schemas") - if err != nil { - return "", err - } - resol := manifests.ResolveOpenApiRefs{} - cache := make(map[string][]byte) - resolved, err := resol.ResolveReferences(byt, definitions, cache) - if err != nil { - return "", err - } - manifest = string(resolved) - return manifest, nil -} diff --git a/utils/component/utils.go b/utils/component/utils.go index f04ded45..f63d6068 100644 --- a/utils/component/utils.go +++ b/utils/component/utils.go @@ -7,6 +7,7 @@ import ( "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/kubernetes" "github.com/layer5io/meshkit/utils/manifests" + "gopkg.in/yaml.v2" ) // Remove the fields which is either not required by end user (like status) or is prefilled by system (like apiVersion, kind and metadata) @@ -80,10 +81,19 @@ func FilterCRDs(manifests [][]byte) ([]string, []error) { var errs []error var filteredManifests []string for _, m := range manifests { - isCrd := kubernetes.IsCRD(string(m)) - if isCrd { - filteredManifests = append(filteredManifests, string(m)) + + var crd map[string]interface{} + err := yaml.Unmarshal(m, &crd) + if err != nil { + errs = append(errs, err) + continue + } + + isCrd := kubernetes.IsCRD(crd) + if !isCrd { + continue } + filteredManifests = append(filteredManifests, string(m)) } return filteredManifests, errs } diff --git a/utils/helm/helm.go b/utils/helm/helm.go index 091e5902..4af65eed 100644 --- a/utils/helm/helm.go +++ b/utils/helm/helm.go @@ -9,7 +9,6 @@ import ( "regexp" "strings" - "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/utils" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" @@ -109,13 +108,7 @@ func writeToFile(w io.Writer, path string) error { if err != nil { return utils.ErrReadFile(err, path) } - - byt, err := encoding.ToYaml(data) - if err != nil { - return utils.ErrWriteFile(err, path) - } - - _, err = w.Write(byt) + _, err = w.Write(data) if err != nil { return utils.ErrWriteFile(err, path) } diff --git a/utils/kubernetes/crd.go b/utils/kubernetes/crd.go index 9bf8d991..0525a624 100644 --- a/utils/kubernetes/crd.go +++ b/utils/kubernetes/crd.go @@ -4,7 +4,6 @@ import ( "context" "github.com/layer5io/meshkit/encoding" - "github.com/layer5io/meshkit/utils" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" ) @@ -54,19 +53,7 @@ func GetGVRForCustomResources(crd *CRDItem) *schema.GroupVersionResource { } } -func IsCRD(manifest string) bool { - cueValue, err := utils.YamlToCue(manifest) - if err != nil { - return false - } - kind, err := utils.Lookup(cueValue, "kind") - if err != nil { - return false - } - kindStr, err := kind.String() - if err != nil { - return false - } - - return kindStr == "CustomResourceDefinition" +func IsCRD(manifest map[string]interface{}) bool { + kind, ok := manifest["kind"].(string) + return ok && kind == "CustomResourceDefinition" } diff --git a/utils/manifests/utils.go b/utils/manifests/utils.go index 2935f1c0..fb3990f7 100644 --- a/utils/manifests/utils.go +++ b/utils/manifests/utils.go @@ -9,7 +9,6 @@ import ( "strings" "cuelang.org/go/cue" - "github.com/layer5io/meshkit/encoding" "github.com/layer5io/meshkit/models/oam/core/v1alpha1" ) @@ -255,7 +254,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue cache = make(map[string][]byte) } var val map[string]interface{} - err := encoding.Unmarshal(manifest, &val) + err := json.Unmarshal(manifest, &val) if err != nil { return nil, err } @@ -267,7 +266,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue if ro.isInsideJsonSchemaProps && (ref == JsonSchemaPropsRef) { // hack so that the UI doesn't crash val["$ref"] = "string" - marVal, errJson := encoding.Marshal(val) + marVal, errJson := json.Marshal(val) if errJson != nil { return manifest, nil } @@ -283,13 +282,13 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue newval = append(newval, v0) continue } - byt, _ := encoding.Marshal(v0) + byt, _ := json.Marshal(v0) byt, err = ro.ResolveReferences(byt, definitions, cache) if err != nil { return nil, err } var newvalmap map[string]interface{} - _ = encoding.Unmarshal(byt, &newvalmap) + _ = json.Unmarshal(byt, &newvalmap) newval = append(newval, newvalmap) } val[k] = newval @@ -334,7 +333,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue if reflect.ValueOf(v).Kind() == reflect.Map { var marVal []byte var def []byte - marVal, err = encoding.Marshal(v) + marVal, err = json.Marshal(v) if err != nil { return nil, err } @@ -350,7 +349,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue } } } - res, err := encoding.Marshal(val) + res, err := json.Marshal(val) if err != nil { return nil, err } @@ -359,7 +358,7 @@ func (ro *ResolveOpenApiRefs) ResolveReferences(manifest []byte, definitions cue func replaceRefWithVal(def []byte, val map[string]interface{}, k string) error { var defVal map[string]interface{} - err := encoding.Unmarshal([]byte(def), &defVal) + err := json.Unmarshal([]byte(def), &defVal) if err != nil { return err }