Merge pull request #148 from FenTiger/metadata

Allow layer author, comment, and created-by metadata to be set

README.md

@@ -269,6 +269,8 @@ Function arguments are:
     building the layer. This is mainly useful to ignore the
     configuration file from the container layer.
 
+- **`metadata`** (defaults to `{ created_by = "nix2container"; }`): an attribute
+    set containing this layer's `created_by`, `author` and `comment` values
 
 ## Isolate dependencies in dedicated layers
 

cmd/layers.go

@@ -17,12 +17,15 @@ import (
 	"github.com/nlewo/nix2container/types"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 var ignore string
 var tarDirectory string
 var permsFilepath string
 var rewritesFilepath string
+var historyFilepath string
 var maxLayers int
 
 // layerCmd represents the layer command
@@ -62,7 +65,16 @@ var layersReproducibleCmd = &cobra.Command{
 				os.Exit(1)
 			}
 		}
-		layers, err := nix.NewLayers(storepaths, maxLayers, parents, rewrites, ignore, perms)
+		var history v1.History
+		if historyFilepath != "" {
+			history, err = readHistoryFile(historyFilepath)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "%s", err)
+				os.Exit(1)
+			}
+		}
+
+		layers, err := nix.NewLayers(storepaths, maxLayers, parents, rewrites, ignore, perms, history)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s", err)
 			os.Exit(1)
@@ -112,7 +124,16 @@ var layersNonReproducibleCmd = &cobra.Command{
 				os.Exit(1)
 			}
 		}
-		layers, err := nix.NewLayersNonReproducible(storepaths, maxLayers, tarDirectory, parents, rewrites, ignore, perms)
+		var history v1.History
+		if historyFilepath != "" {
+			history, err = readHistoryFile(historyFilepath)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "%s", err)
+				os.Exit(1)
+			}
+		}
+
+		layers, err := nix.NewLayersNonReproducible(storepaths, maxLayers, tarDirectory, parents, rewrites, ignore, perms, history)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s", err)
 			os.Exit(1)
@@ -157,12 +178,14 @@ func init() {
 
 	layersNonReproducibleCmd.Flags().StringVarP(&rewritesFilepath, "rewrites", "", "", "A JSON file containing a list of path rewrites. Each element of the list is a JSON object with the attributes path, regex and repl: for a given path, the regex is replaced by repl.")
 	layersNonReproducibleCmd.Flags().StringVarP(&permsFilepath, "perms", "", "", "A JSON file containing file permissions")
+	layersNonReproducibleCmd.Flags().StringVarP(&historyFilepath, "history", "", "", "A JSON file containing layer history")
 	layersNonReproducibleCmd.Flags().IntVarP(&maxLayers, "max-layers", "", 1, "The maximum number of layers")
 
 	rootCmd.AddCommand(layersReproducibleCmd)
 	layersReproducibleCmd.Flags().StringVarP(&ignore, "ignore", "", "", "Ignore the path from the list of storepaths")
 	layersReproducibleCmd.Flags().StringVarP(&rewritesFilepath, "rewrites", "", "", "A JSON file containing path rewrites")
 	layersReproducibleCmd.Flags().StringVarP(&permsFilepath, "perms", "", "", "A JSON file containing file permissions")
+	layersReproducibleCmd.Flags().StringVarP(&historyFilepath, "history", "", "", "A JSON file containing layer history")
 	layersReproducibleCmd.Flags().IntVarP(&maxLayers, "max-layers", "", 1, "The maximum number of layers")
 
 }

cmd/utils.go

@@ -5,6 +5,8 @@ import (
 	"os"
 
 	"github.com/nlewo/nix2container/types"
+
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 func readPermsFile(filename string) (permPaths []types.PermPath, err error) {
@@ -30,3 +32,15 @@ func readRewritesFile(filename string) (rewritePaths []types.RewritePath, err er
 	}
 	return
 }
+
+func readHistoryFile(filename string) (history v1.History, err error) {
+	content, err := os.ReadFile(filename)
+	if err != nil {
+		return history, err
+	}
+	err = json.Unmarshal(content, &history)
+	if err != nil {
+		return history, err
+	}
+	return
+}

default.nix

@@ -260,6 +260,8 @@ let
     maxLayers ? 1,
     # Deprecated: will be removed on v1
     contents ? null,
+    # Author, comment, created_by
+    metadata ? { created_by = "nix2container"; },
   }: let
     subcommand = if reproducible
               then "layers-from-reproducible-storepaths"
@@ -282,6 +284,8 @@ let
     rewritesFlag = "--rewrites ${rewritesFile}";
     permsFile = pkgs.writeText "perms.json" (l.toJSON perms);
     permsFlag = l.optionalString (perms != []) "--perms ${permsFile}";
+    historyFile = pkgs.writeText "history.json" (l.toJSON metadata);
+    historyFlag = l.optionalString (metadata != {}) "--history ${historyFile}";
     allDeps = deps ++ copyToRootList;
     tarDirectory = l.optionalString (! reproducible) "--tar-directory $out";
     layersJSON = pkgs.runCommand "layers.json" {} ''
@@ -292,6 +296,7 @@ let
         --max-layers ${toString maxLayers} \
         ${rewritesFlag} \
         ${permsFlag} \
+        ${historyFlag} \
         ${tarDirectory} \
         ${l.concatMapStringsSep " "  (l: l + "/layers.json") layers} \
       '';

examples/default.nix

@@ -15,4 +15,5 @@
   nix-user = pkgs.callPackage ./nix-user.nix { inherit nix2container; };
   ownership = pkgs.callPackage ./ownership.nix { inherit nix2container; };
   created = pkgs.callPackage ./created.nix { inherit nix2container; };
+  metadata = pkgs.callPackage ./metadata.nix { inherit nix2container; };
 }

examples/metadata.nix

@@ -0,0 +1,17 @@
+{ pkgs, nix2container }:
+nix2container.buildImage {
+  name = "metadata";
+  config = {
+    entrypoint = ["${pkgs.hello}/bin/hello"];
+  };
+  layers = [
+    (nix2container.buildLayer {
+      deps = [ pkgs.hello ];
+      metadata = {
+        created_by = "test created_by";
+        author = "test author";
+        comment = "test comment";
+      };
+    })
+  ];
+}

nix/image.go

@@ -84,15 +84,14 @@ func getV1Image(image types.Image) (imageV1 v1.Image, err error) {
 			imageV1.RootFS.DiffIDs,
 			digest)
 		imageV1.RootFS.Type = "layers"
+		// Even if optional in the spec, we
+		// need to add an history otherwise
+		// some toolings can complain:
+		// https://github.com/nlewo/nix2container/issues/57
 		imageV1.History = append(
 			imageV1.History,
-			v1.History{
-				// Even if optional in the spec, we
-				// need to add an history otherwise
-				// some toolings can complain:
-				// https://github.com/nlewo/nix2container/issues/57
-				CreatedBy: "nix2container",
-			})
+			layer.History,
+		)
 	}
 	return
 }

nix/image_test.go

@@ -38,6 +38,9 @@ func TestGetV1Image(t *testing.T) {
 				DiffIDs:   "sha256:adf74a52f9e1bcd7dab77193455fa06743b979cf5955148010e5becedba4f72d",
 				Size:      10,
 				MediaType: "application/vnd.oci.image.layer.v1.tar",
+				History: v1.History{
+					CreatedBy: "nix2container",
+				},
 			},
 		},
 	}

nix/layers.go

@@ -64,7 +64,7 @@ func getPaths(storePaths []string, parents []types.Layer, rewrites []types.Rewri
 // If tarDirectory is not an empty string, the tar layer is written to
 // the disk. This is useful for layer containing non reproducible
 // store paths.
-func newLayers(paths types.Paths, tarDirectory string, maxLayers int) (layers []types.Layer, err error) {
+func newLayers(paths types.Paths, tarDirectory string, maxLayers int, history v1.History) (layers []types.Layer, err error) {
 	offset := 0
 	for offset < len(paths) {
 		max := offset + 1
@@ -90,6 +90,7 @@ func newLayers(paths types.Paths, tarDirectory string, maxLayers int) (layers []
 			Size:      size,
 			Paths:     layerPaths,
 			MediaType: v1.MediaTypeImageLayer,
+			History:   history,
 		}
 		if tarDirectory != "" {
 			// TODO: we should use v1.MediaTypeImageLayerGzip instead
@@ -104,14 +105,14 @@ func newLayers(paths types.Paths, tarDirectory string, maxLayers int) (layers []
 	return layers, nil
 }
 
-func NewLayers(storePaths []string, maxLayers int, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath) ([]types.Layer, error) {
+func NewLayers(storePaths []string, maxLayers int, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath, history v1.History) ([]types.Layer, error) {
 	paths := getPaths(storePaths, parents, rewrites, exclude, perms)
-	return newLayers(paths, "", maxLayers)
+	return newLayers(paths, "", maxLayers, history)
 }
 
-func NewLayersNonReproducible(storePaths []string, maxLayers int, tarDirectory string, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath) (layers []types.Layer, err error) {
+func NewLayersNonReproducible(storePaths []string, maxLayers int, tarDirectory string, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath, history v1.History) (layers []types.Layer, err error) {
 	paths := getPaths(storePaths, parents, rewrites, exclude, perms)
-	return newLayers(paths, tarDirectory, maxLayers)
+	return newLayers(paths, tarDirectory, maxLayers, history)
 }
 
 func isPathInLayers(layers []types.Layer, path types.Path) bool {

nix/layers_test.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/nlewo/nix2container/types"
 	"github.com/stretchr/testify/assert"
+
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 func TestPerms(t *testing.T) {
@@ -18,7 +20,7 @@ func TestPerms(t *testing.T) {
 			Mode:  "0641",
 		},
 	}
-	layer, err := NewLayers(paths, 1, []types.Layer{}, []types.RewritePath{}, "", perms)
+	layer, err := NewLayers(paths, 1, []types.Layer{}, []types.RewritePath{}, "", perms, v1.History{})
 	if err != nil {
 		t.Fatalf("%v", err)
 	}
@@ -50,7 +52,7 @@ func TestNewLayers(t *testing.T) {
 	paths := []string{
 		"../data/layer1/file1",
 	}
-	layer, err := NewLayers(paths, 1, []types.Layer{}, []types.RewritePath{}, "", []types.PermPath{})
+	layer, err := NewLayers(paths, 1, []types.Layer{}, []types.RewritePath{}, "", []types.PermPath{}, v1.History{})
 	if err != nil {
 		t.Fatalf("%v", err)
 	}
@@ -70,7 +72,7 @@ func TestNewLayers(t *testing.T) {
 	assert.Equal(t, expected, layer)
 
 	tmpDir := t.TempDir()
-	layer, err = NewLayersNonReproducible(paths, 1, tmpDir, []types.Layer{}, []types.RewritePath{}, "", []types.PermPath{})
+	layer, err = NewLayersNonReproducible(paths, 1, tmpDir, []types.Layer{}, []types.RewritePath{}, "", []types.PermPath{}, v1.History{})
 	if err != nil {
 		t.Fatalf("%v", err)
 	}

tests/default.nix

@@ -155,6 +155,45 @@ let
         exit $ret
       fi
     '';
+    metadata = let
+      image = examples.metadata;
+      expected_created_by = "test created_by";
+      expected_author = "test author";
+      expected_comment = "test comment";
+    in pkgs.writeScriptBin "test-script" ''
+      ${image.copyToPodman}/bin/copy-to-podman
+      created_by=$(${pkgs.podman}/bin/podman image inspect ${image.imageName}:${image.imageTag} -f '{{ (index .History 0).CreatedBy }}')
+      author=$(${pkgs.podman}/bin/podman image inspect ${image.imageName}:${image.imageTag} -f '{{ (index .History 0).Author }}')
+      comment=$(${pkgs.podman}/bin/podman image inspect ${image.imageName}:${image.imageTag} -f '{{ (index .History 0).Comment }}')
+      if ! echo $created_by | ${pkgs.gnugrep}/bin/grep '${expected_created_by}' > /dev/null;
+      then
+        echo "Expected created_by attribute to contain: ${expected_created_by}"
+        echo ""
+        echo "Actual created_by attribute: $created"
+        echo ""
+        echo "Error: test failed"
+        exit 1
+      fi
+      if ! echo $author | ${pkgs.gnugrep}/bin/grep '${expected_author}' > /dev/null;
+      then
+        echo "Expected author attribute to contain: ${expected_author}"
+        echo ""
+        echo "Actual author attribute: $author"
+        echo ""
+        echo "Error: test failed"
+        exit 1
+      fi
+      if ! echo $comment | ${pkgs.gnugrep}/bin/grep '${expected_comment}' > /dev/null;
+      then
+        echo "Expected comment attribute to contain: ${expected_comment}"
+        echo ""
+        echo "Actual comment attribute: $comment"
+        echo ""
+        echo "Error: test failed"
+        exit 1
+      fi
+      echo "Test passed"
+    '';
   } //
   (pkgs.lib.mapAttrs' (name: drv: {
     name = "${name}GetManifest";
@@ -173,8 +212,8 @@ let
       "\n"
       (pkgs.lib.mapAttrsToList (n: v: "echo Running test '${n}'\n${v}/bin/test-script") tests);
     in pkgs.writeScriptBin "all-test-scripts" ''
+      #!${pkgs.runtimeShell}
       set -e
       ${scripts}
     '';
 in tests // { inherit all; }
-

types/types.go

@@ -81,6 +81,7 @@ type Layer struct {
 	// https://github.com/opencontainers/image-spec/blob/8b9d41f48198a7d6d0a5c1a12dc2d1f7f47fc97f/specs-go/v1/mediatype.go
 	MediaType string `json:"mediatype"`
 	LayerPath string `json:"layer-path,omitempty"`
+	History   v1.History
 }
 
 func NewLayersFromFile(filename string) ([]Layer, error) {