Benchmarking

Makefile

@@ -144,6 +144,13 @@ feature_%: ## Run acceptance tests for a single feature file, e.g. make feature_
 scenario_%: build ## Run acceptance tests for a single scenario, e.g. make scenario_inline_policy
 	@cd acceptance && go test -test.run 'TestFeatures/$*'
 
+benchmark_%:
+	cd benchmark/$*
+	go run . -benchnum 10
+
+.PHONY: benchmark
+benchmark: benchmark_simple ## Run benchmarks
+
 .PHONY: ci
 ci: test lint-fix acceptance ## Run the usual required CI tasks
 

benchmark/README.md

@@ -0,0 +1,12 @@
+# Benchmarks of ec CLI
+
+Benchmarks within this directory use the [golang
+benchmarking](golang.org/x/benchmarks/) package and output in the [standard
+benchmark
+format](https://go.googlesource.com/proposal/+/master/design/14313-benchmark-format.md).
+
+Each benchmark is built as a standalone executable with no external dependency
+other than any data that is contained within it. Benchmarks are run from within
+the directory they're defined in, simply by running `go run .`, additional
+arguments can be passed in, for example `-benchnum 10` to run the benchmark 10
+times.

benchmark/internal/registry/registry.go

@@ -0,0 +1,50 @@
+// Copyright The Enterprise Contract Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package registry
+
+import (
+	"context"
+
+	"github.com/testcontainers/testcontainers-go/modules/registry"
+)
+
+type Closer interface {
+	Close()
+}
+
+type registryCloser struct {
+	container *registry.RegistryContainer
+}
+
+func (r *registryCloser) Close() {
+	if r == nil || r.container == nil {
+		return
+	}
+
+	_ = r.container.Terminate(context.Background())
+}
+
+func Launch(data string) (string, Closer, error) {
+	ctx := context.Background()
+	r, err := registry.Run(ctx, "registry:2.8.3", registry.WithData(data))
+	c := &registryCloser{r}
+	if err != nil {
+		return "", c, err
+	}
+
+	return r.RegistryName, c, nil
+}

benchmark/internal/suite/suite.go

@@ -0,0 +1,32 @@
+// Copyright The Enterprise Contract Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suite
+
+import (
+	"io"
+
+	"github.com/enterprise-contract/ec-cli/cmd"
+	"github.com/enterprise-contract/ec-cli/cmd/root"
+)
+
+func Execute(args []string) error {
+	c := root.NewRootCmd()
+	cmd.AddCommandsTo(c)
+	c.SetArgs(args)
+	c.SetOutput(io.Discard)
+	return c.Execute()
+}

benchmark/offliner/README.md

@@ -0,0 +1,8 @@
+# Offliner
+
+A tool to offline, i.e. place all data for an container image in a remote
+registry to a local data directory. This local data directory can be mounted to
+docker.io/registry container registry so running against it eliminates remote
+network access.
+
+Use: `offliner <pinned image reference> <data directory>`

benchmark/offliner/offliner.go

@@ -0,0 +1,213 @@
+// Copyright The Enterprise Contract Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/google/go-containerregistry/pkg/logs"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/modules/registry"
+)
+
+func items(sourceImages []name.Reference, destination name.Registry) map[string]string {
+	items := map[string]string{}
+	for _, sourceImage := range sourceImages {
+		var dest fmt.Stringer = destination.Repo(sourceImage.Context().RepositoryStr())
+		if d, ok := sourceImage.(name.Digest); ok {
+			dest = dest.(name.Repository).Digest(d.DigestStr())
+		}
+		if t, ok := sourceImage.(name.Tag); ok {
+			dest = dest.(name.Repository).Tag(t.TagStr())
+		}
+
+		items[sourceImage.String()] = dest.String()
+	}
+
+	return items
+}
+
+func digestTag(ref name.Reference, suffix string) name.Reference {
+	digest := ref.(name.Digest).DigestStr()
+
+	tag := fmt.Sprintf("%s.%s", strings.Replace(digest, ":", "-", 1), suffix)
+
+	return ref.Context().Tag(tag)
+}
+
+func signature(ref name.Reference) name.Reference {
+	return digestTag(ref, "sig")
+}
+
+func attestation(ref name.Reference) name.Reference {
+	return digestTag(ref, "att")
+}
+
+func sbom(ref name.Reference) name.Reference {
+	return digestTag(ref, "sbom")
+}
+
+func referrers(ref name.Reference) ([]name.Reference, error) {
+	var err error
+	var idx v1.ImageIndex
+	if d, ok := ref.(name.Digest); ok {
+		idx, err = remote.Referrers(d)
+
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		img, err := remote.Image(ref)
+		if err != nil {
+			return nil, err
+		}
+
+		digest, err := img.Digest()
+		if err != nil {
+			return nil, err
+		}
+		ref := ref.Context().Digest(digest.String())
+		idx, err = remote.Referrers(ref)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	indexManifest, err := idx.IndexManifest()
+	if err != nil {
+		return nil, err
+	}
+
+	references := make([]name.Reference, 0, len(indexManifest.Manifests))
+	for _, manifest := range indexManifest.Manifests {
+		r := ref.Context().Digest(manifest.Digest.String())
+		references = append(references, r)
+	}
+
+	return references, nil
+}
+
+func related(ref name.Reference) []name.Reference {
+	references := []name.Reference{ref}
+
+	for _, r := range []name.Reference{
+		signature(ref),
+		attestation(ref),
+		sbom(ref),
+	} {
+		if _, err := remote.Image(r); err == nil {
+			references = append(references, r)
+		}
+	}
+
+	refs, err := referrers(ref)
+	if err != nil {
+		return references
+	}
+
+	return append(references, refs...)
+}
+
+func main() {
+	if len(os.Args) != 3 {
+		fmt.Fprintf(os.Stderr, "Usage: %s <image> <directory>\n", os.Args[0])
+		os.Exit(1)
+	}
+
+	sourceImage := os.Args[1]
+	dir, err := filepath.Abs(os.Args[2])
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(2)
+	}
+
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(3)
+		}
+	}
+
+	source, err := name.ParseReference(sourceImage)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(4)
+	}
+
+	_, ok := source.(name.Digest)
+	if !ok {
+		fmt.Fprintln(os.Stderr, "use pinned image references")
+		os.Exit(5)
+	}
+
+	ctx := context.Background()
+	registry, err := registry.Run(ctx,
+		registry.DefaultImage,
+		testcontainers.WithConfigModifier(func(config *container.Config) {
+			config.User = fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid())
+		}),
+		testcontainers.WithHostConfigModifier(func(hostConfig *container.HostConfig) {
+			hostConfig.UsernsMode = "host"
+			hostConfig.Binds = append(hostConfig.Binds, fmt.Sprintf("%s:/var/lib/registry:z", dir))
+		}),
+	)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(6)
+	}
+	defer registry.Terminate(ctx)
+
+	destination, err := name.NewRegistry(registry.RegistryName, name.Insecure)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(7)
+	}
+	logs.Progress = log.New(os.Stdout, "", log.LstdFlags)
+
+	sourceImages := related(source)
+
+	idx, err := remote.Index(source)
+	if err == nil {
+		idxManifest, err := idx.IndexManifest()
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(8)
+		}
+
+		for _, m := range idxManifest.Manifests {
+			img := source.Context().Digest(m.Digest.String())
+			sourceImages = append(sourceImages, related(img)...)
+		}
+	}
+
+	for s, d := range items(sourceImages, destination) {
+		if err := crane.Copy(s, d); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(9)
+		}
+	}
+}

benchmark/offliner/offliner_test.go

@@ -0,0 +1,49 @@
+// Copyright The Enterprise Contract Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"maps"
+	"testing"
+
+	"github.com/google/go-containerregistry/pkg/name"
+)
+
+func TestItems(t *testing.T) {
+	dest, err := name.NewRegistry("newregistry.io")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	imgs := []name.Reference{
+		name.MustParseReference("registry.io/repository/image"),
+		name.MustParseReference("registry.io/repository/image:tag"),
+		name.MustParseReference("registry.io/repository/image@sha256:9f80b4f7506d2799298a685162723482cac160abf701e029ee5cbaa6c74967ea"),
+	}
+
+	got := items(imgs, dest)
+
+	expected := map[string]string{
+		"registry.io/repository/image":     "newregistry.io/repository/image:latest",
+		"registry.io/repository/image:tag": "newregistry.io/repository/image:tag",
+		"registry.io/repository/image@sha256:9f80b4f7506d2799298a685162723482cac160abf701e029ee5cbaa6c74967ea": "newregistry.io/repository/image@sha256:9f80b4f7506d2799298a685162723482cac160abf701e029ee5cbaa6c74967ea",
+	}
+
+	if !maps.Equal(expected, got) {
+		t.Errorf("expected and got images differ: %v != %v", expected, got)
+	}
+}

benchmark/simple/data/registry/data/docker/registry/v2/blobs/sha256/0f/0f61be8d87847a8ae7c5a52608e67cc37ec565d2be274e50bfeafea5d2d3b154/data

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","size":342,"digest":"sha256:2b5c9030c3dc80a3cc7c1e2f6454b9b8dc534f5ef4594b1264d8999adbb9acdc"},"layers":[{"mediaType":"application/vnd.dev.cosign.simplesigning.v1+json","size":301,"digest":"sha256:d1317b7f65c749d2d588bc8df2ee25d35f0e6049f8f988d8a12e9761eaede92f","annotations":{"dev.cosignproject.cosign/signature":"MEUCIF8OHzZbAgy8LcWzaOfDdFhngP3Dc1L219CKzCP5p0/DAiEA5ZLbmY0b0YwdpBIekcgs5ZhjL58vl4Z5UaIc7RfB1+s=","dev.sigstore.cosign/certificate":"","dev.sigstore.cosign/chain":""}},{"mediaType":"application/vnd.dev.cosign.simplesigning.v1+json","size":301,"digest":"sha256:d1317b7f65c749d2d588bc8df2ee25d35f0e6049f8f988d8a12e9761eaede92f","annotations":{"dev.cosignproject.cosign/signature":"MEYCIQD36/rc9PNzQfxmQxCNBurLUE3l7H897d9TQxL44TpM/AIhAPze0wYsIcYNfD9kL115AXVz+Pm5H0ecUMYCyOeQzNN6","dev.sigstore.cosign/certificate":"","dev.sigstore.cosign/chain":""}}]}