Provide a function that makes it easier to join sets of different systems

[NobbZ]

I have a flake that provides a docker image as package in its output.

All but docker image are buildable and should run on the "default systems", so I wanted to move the docker image out of the function into its own attribute set which I then merge them, though due to the way // works, one packages overwrites the other:

outputs = { … }
  {
    packages.x86_64-linux.image = …;
  } // eachDefaultSystem (system:
  {
    packages.somethingElse = …;
  }

will create a flake only provides somethingElse for the default systems, if you turn it around to be

outputs = { … }
  eachDefaultSystem (system:
  {
    packages.somethingElse = …;
  }) // {
    packages.x86_64-linux.image = …;
  }

You will end up with the x86_64-linux.image only.

a mergeOutputs could take a list of attribute sets thats then gets deepmerged, usage would be roughly:

outputs = { … }
  mergeOutputs [
    (eachDefaultSystem (system:
    {
      packages.somethingElse = …;
    }))
    ({
      packages.x86_64-linux.image = …;
    })
  ];

[zimbatm]

What do you think of this approach?

{
  outputs = { nixpkgs, flake-utils, ... }:
    flake-utils.lib.eachDefaultSystem (system:
      (nixpkgs.lib.optionalAttrs (system == "x86_64-linux") {
        image = <drv>;
      }) // {
        somethingElse = <drv>;
      });
}

I think it's better because it doesn't expose the packages.<system> construct to the reader.

[NobbZ]

Thats a great idea!

[zimbatm]

Okay, in that case there isn't much to do. Maybe document the design pattern somewhere, not sure where though.

[pingiun]

This helped me, I think it would be nice to have in the README.md

[chasecaleb]

I have a new variation of this that's a little more complex, because I have other derivations that need to refer to the Linux-specific Docker image derivation. Here's what I came up with, but it's pretty atrocious:

  outputs = { self, nixpkgs, flakeUtils }:
    let
      systemOutputs = flakeUtils.lib.eachDefaultSystem
        (system:
          let
            pkgs = nixpkgs.legacyPackages.${system};
            devPackages = [
              pkgs.coreutils-full
              pkgs.curl
              pkgs.jq
            ];
          in
          rec {
            packages = pkgs.lib.optionalAttrs (system == "x86_64-linux") {
              image = pkgs.dockerTools.buildLayeredImage {
                name = "hello-nix";
                contents = devPackages;
                config = {
                  Cmd = [ "${pkgs.bashInteractive}/bin/bash" ];
                };
              };
            };

            devShell = devShells.base;
            devShells = {
              base = pkgs.mkShell {
                packages = devPackages;
              };
            };
          });
    in
    # This needs to be separate to allow referring to the linux version of packages.image.
    systemOutputs // flakeUtils.lib.eachDefaultSystem (system:
      let
        # Even on Mac we still build and run a Linux Docker image.
        image = systemOutputs.packages.x86_64-linux.image;
        pkgs = nixpkgs.legacyPackages.${system};
      in
      rec {
        defaultPackage = image;

        apps = {
          # packages.image builds a docker image as a tar.gz, so here's a quick wrapper script that
          # loads it into your local docker and runs it.
          dockerApp = pkgs.writeShellScriptBin "run-docker" ''
            echo "Loading docker image from ${image}"
            sudo docker load < "${image}"
            sudo docker run --rm -it "${image.imageName}:${image.imageTag}"
          '';
        };
        defaultApp = apps.dockerApp;
      });

As you can see it makes it so that the ${image} usage in outputs.apps.x86_64-darwin.dockerApp refers to packages.x86_64-linux.image. Is there a cleaner way to do this?

[chasecaleb]

Alright, after nearly giving up I did some refactoring to shuffle things around and I still don't love this, but I think it's reasonable-ish. Certainly better than my previous comment.

  outputs = { self, nixpkgs, flakeUtils }:
    let
      devPackages = p: [
        p.coreutils-full
        p.curl
        p.jq
        p.cowsay
      ];
      linuxPkgs = nixpkgs.legacyPackages.x86_64-linux;
      image = linuxPkgs.dockerTools.buildLayeredImage {
        name = "hello-nix";
        contents = devPackages linuxPkgs;
        config = {
          Cmd = [ "${linuxPkgs.bashInteractive}/bin/bash" ];
        };
      };
    in
    flakeUtils.lib.eachDefaultSystem (system:
      let
        pkgs = nixpkgs.legacyPackages.${system};
      in
      rec {
        packages = (pkgs.lib.optionalAttrs (system == "x86_64-linux") {
          image = image;
        }) // {
          # Other packages (built on any system) would go here.
        };
        # Even on Mac we still but the Linux version of a Docker image.
        defaultPackage = image;

        apps = {
          dockerApp = pkgs.writeShellScriptBin "run-docker" ''
            echo "Loading docker image from ${image}"
            sudo docker load < "${image}"
            sudo docker run --rm -it "${image.imageName}:${image.imageTag}"
          '';
        };
        defaultApp = apps.dockerApp;

        devShell = devShells.base;
        devShells = {
          base = pkgs.mkShell {
            packages = devPackages pkgs;
          };
        };
      });

If anyone has better approaches it would be appreciated!

[sacundim]

@chasecaleb I just independently retraced a bunch of what you did, I'm still trying to get a feel for this stuff too. Simplifying some details away:

  outputs = { self, nixpkgs, flake-utils, poetry2nix }:
    let
      # Utility function to select systems by CPU arch or OS 
      filterSystems = pred:
        let
          inherit (builtins) elemAt filter isList match;
          candidates = flake-utils.lib.defaultSystems;
          analyze = system:
            let matches = match "([[:alnum:]_]+)-([[:alnum:]_]+)" system;
            in { arch = elemAt matches 0; os = elemAt matches 1; };
        in
          filter (system: pred (analyze system)) candidates;

      baseOutputs = flake-utils.lib.eachDefaultSystem (system:
        let
          [...]
        in rec {
          packages = {
            downloader = [...];
            website = [...];
          };
        }
      );

      linuxSystems = filterSystems (sys: sys.os == "linux");
      dockerImages = flake-utils.lib.eachSystem linuxSystems (system:
        let
          pkgs = nixpkgs.legacyPackages.${system}.pkgs;
        in {
          packages = {
            downloader-docker = pkgs.dockerTools.streamLayeredImage {
              contents = [ baseOutputs.packages.${system}.downloader ];
              name = [...]; tag = [...];
            };

            website-docker = [...];
          };
        }
      );

      # Another aux function, to merge two whole output sets. The key insight
      # is that we only merge recursively down to two levels.
      mergeOutputs =
        let
          inherit (builtins) length;
          inherit (nixpkgs.lib.attrsets) recursiveUpdateUntil;
          mergeDepth = depth:
            recursiveUpdateUntil (path: l: r: length path > depth);
        in builtins.foldl' (mergeDepth 2) {};

    in mergeOutputs [
        baseOutputs
        dockerImages
    ];

I get this output tree for my actual (a bit more complex) flake

% nix flake show
git+file:///Users/sacundim/Code/covid-19-puerto-rico?ref=refs%2fheads%2fnix-flake&rev=ec2b9752fcd6f83d2c5009ab3c214b67c0dd5adc
└───packages
    ├───aarch64-darwin
    │   ├───default: package 'covid-19-puerto-rico'
    │   ├───downloader: package 'python3.11-covid-19-puerto-rico-downloader'
    │   └───website: package 'python3.11-covid-19-puerto-rico-website'
    ├───aarch64-linux
    │   ├───default: package 'covid-19-puerto-rico'
    │   ├───downloader: package 'python3.11-covid-19-puerto-rico-downloader'
    │   ├───downloader-docker: package 'stream-covid-19-puerto-rico-downloader'
    │   ├───website: package 'python3.11-covid-19-puerto-rico-website'
    │   └───website-docker: package 'stream-covid-19-puerto-rico-website'
    ├───x86_64-darwin
    │   ├───default: package 'covid-19-puerto-rico'
    │   ├───downloader: package 'python3.11-covid-19-puerto-rico-downloader'
    │   └───website: package 'python3.11-covid-19-puerto-rico-website'
    └───x86_64-linux
        ├───default: package 'covid-19-puerto-rico'
        ├───downloader: package 'python3.11-covid-19-puerto-rico-downloader'
        ├───downloader-docker: package 'stream-covid-19-puerto-rico-downloader'
        ├───website: package 'python3.11-covid-19-puerto-rico-website'
        └───website-docker: package 'stream-covid-19-puerto-rico-website'

EDIT: Fixed a mistake in the code