nixpkgs
nixpkgs copied to clipboard
NixOS
ERROR: Could not find a version that satisfies the requirement packaging (from > ERROR: No matching distribution found for packaging
Steps To Reproduce
Steps to reproduce the behavior:
# flake.nix
{
inputs = {
nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
utils.url = "github:numtide/flake-utils";
};
outputs = {
self,
nixpkgs,
utils,
}:
utils.lib.eachDefaultSystem (system: let
pkgs = import nixpkgs {inherit system;};
python = pkgs.python39.override {
packageOverrides = pyfinal: pyprev: rec {
clu = pyprev.buildPythonPackage rec {
pname = "clu";
version = "0.0.7";
src = builtins.fetchTarball {
sha256 = "sha256:0scpcw5vylg2f3fd8hgwr6zkcwl8kwq1m16p3yh6dqrcay3xx38p";
url = "https://files.pythonhosted.org/packages/fc/dc/8dbb7f562c9d684b78be44b091d277775461dcdc9f3fce3732674c89b36b/clu-0.0.7.tar.gz";
};
};
};
};
in {
devShell = pkgs.mkShell {
buildInputs = [python.pkgs.clu];
};
packages.default = python.withPackages (p: [p.clu]);
});
}
nix --extra-experimental-features flakes build
Build log
https://gist.github.com/ethanabrooks/17f81c76e540e7e2581920690aaf0a47
Additional context
I am trying to add the clu library which exists in Pypi but seems to have been packaged weirdly. Thank you for your help!
Also, would it be possible to provide a pointer to the buildPythonPackage source code? I could not find it in the repo.
Notify maintainers
Metadata
Please run nix-shell -p nix-info --run "nix-info -m" and paste the result.
[user@system:~]$ nix-shell -p nix-info --run "nix-info -m"
this path will be fetched (0.00 MiB download, 0.00 MiB unpacked):
/nix/store/l1993ah8cpzz300p2dmpk7j5nmp4ahq3-nix-info
copying path '/nix/store/l1993ah8cpzz300p2dmpk7j5nmp4ahq3-nix-info' from 'https://cache.nixos.org'...
- system: `"x86_64-linux"`
- host os: `Linux 5.4.0-120-generic, Ubuntu, 20.04.4 LTS (Focal Fossa), nobuild`
- multi-user?: `yes`
- sandbox: `yes`
- version: `nix-env (Nix) 2.10.3`
- channels(ethanbro): `"home-manager"`
- channels(root): `"nixpkgs"`
- nixpkgs: `/nix/var/nix/profiles/per-user/root/channels/nixpkgs`
You need to specify all the dependencies of clu: https://github.com/google/CommonLoopUtils/blob/main/setup.py#L46
See 17.27.1.2.2. Handling dependencies in the manual: https://nixos.org/manual/nixpkgs/unstable/
As argument to buildPythonPackage:
propagatedBuildInputs = [ pyprev.flax ... ];
Since this is not currently in nixpkgs, this is not a bug or build failure. If you would like to see it in nixpkgs, open an issue with the "Packaging request" template.
To get something working quickly you might have better luck with Python tools like:
https://github.com/DavHau/mach-nix
https://github.com/nix-community/poetry2nix
Hi iwanb thanks for the quick response. Since clu has so many dependencies, some of which are quite large, I am going to try to go back to poetry2nix for this. I will post a solution when I have one.
I am actually very close to a working implementation. The following works for me:
{
inputs = {
nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
utils.url = "github:numtide/flake-utils";
};
outputs = {
self,
nixpkgs,
utils,
}: let
out = system: let
pkgs = import nixpkgs {
inherit system;
config = {
allowUnfree = true;
cudaSupport = true;
};
};
inherit (pkgs) poetry2nix;
inherit (poetry2nix) mkPoetryApplication mkPoetryEnv;
inherit (pkgs.cudaPackages) cudatoolkit;
inherit (pkgs.linuxPackages) nvidia_x11;
python = pkgs.python39;
overrides = pyfinal: pyprev: rec {
clu = pyprev.buildPythonPackage rec {
pname = "clu";
version = "0.0.7";
src = pyprev.fetchPypi {
inherit pname version;
sha256 = "sha256-RJqa8XnDpcRPwYlH+4RKAOos0x4+3hMWf/bv6JNn2ys=";
};
buildInputs = with pyfinal; [
absl-py
etils
flax
jax
jaxlib
ml-collections
numpy
packaging
tensorflow
tensorflow-datasets
];
};
ml-collections =
pyprev.buildPythonPackage
rec {
pname = "ml_collections";
version = "0.1.1";
src = pyprev.fetchPypi {
inherit pname version;
sha256 = "sha256-P+/McuxDOqHl0yMHo+R0u7Z/QFvoFOpSohZr/J2+aMw=";
};
buildInputs = with pyfinal; [
absl-py
contextlib2
pyyaml
six
];
prePatch = ''
export HOME=$TMPDIR;
'';
};
tensorflow-gpu =
# Override the nixpkgs bin version instead of
# poetry2nix version so that rpath is set correctly.
pyprev.tensorflow-bin.overridePythonAttrs
(old: {inherit (pyprev.tensorflow-gpu) src version;});
astunparse = pyprev.astunparse.overridePythonAttrs (old: {
buildInputs = (old.buildInputs or []) ++ [pyfinal.wheel];
});
# Use cuda-enabled jaxlib as required
jaxlib = pyprev.jaxlibWithCuda.override {
inherit
(pyprev)
absl-py
flatbuffers
numpy
scipy
six
;
};
};
poetryArgs = {
inherit python;
projectDir = ./.;
preferWheels = true;
overrides = poetry2nix.overrides.withDefaults overrides;
};
poetryEnv = mkPoetryEnv poetryArgs;
in {
devShell = pkgs.mkShell rec {
buildInputs = with pkgs; [
cudatoolkit
nvidia_x11
poetry
poetryEnv
pre-commit
];
shellHook = ''
export PYTHONFAULTHANDLER=1
export PYTHONBREAKPOINT=ipdb.set_trace
set -o allexport
source .env
set +o allexport
export CUDA_PATH=${cudatoolkit.lib}
export LD_LIBRARY_PATH=${cudatoolkit.lib}/lib:${nvidia_x11}/lib
export EXTRA_LDFLAGS="-l/lib -l${nvidia_x11}/lib"
export EXTRA_CCFLAGS="-i/usr/include"
'';
};
packages.default = mkPoetryApplication poetryArgs;
};
in
utils.lib.eachDefaultSystem out;
}
This is my pyproejct.toml:
[tool.poetry]
name = "ppo"
version = "0.1.0"
description = ""
authors = ["Logan Walls <[email protected]>", "Ethan Brooks <[email protected]>"]
[tool.poetry.dependencies]
python = ">=3.8,<3.11"
jaxlib = "0.3.0"
jax = "0.3.6"
gym-minigrid = "^1.1.0"
dollar-lambda = "^1.1.4"
run-logger = "^0.1.8"
tensorflow-gpu = "^2.9.1"
flax = "^0.5.3"
[tool.poetry.dev-dependencies]
[tool.poetry.scripts]
main = 'ppo:main'
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
However when I add clu (poetry add clu --lock), I first have to delete this line in poetry.lock:
etils = {version = "*", extras = ["epy"], optional = true, markers = "extra == \"epath\""}
because etils depends on itself and this causes an infinite recursion.
Next I get this error:
error: builder for '/nix/store/h76j40mzmc0y2bcy50v4lqfigynn32pd-python3.9-ml_collections-0.1.1.drv' failed with exit code 1;
last 10 log lines:
> /nix/store/cnnykza37klyikkzb2dwjh1kbhvyhrc9-python3.9-setuptools-61.2.0/lib/python3.9/site-packages/setuptools/installer.py:27: SetuptoolsDeprecationWarning: setuptools.installer is deprecated. Requirements should be satisfied by a PEP 517 installer.
> warnings.warn(
> WARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x7ffff580e4f0>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution')': /simple/mock/
> WARNING: Retrying (Retry(total=3, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x7ffff580e880>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution')': /simple/mock/
> WARNING: Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x7ffff580eaf0>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution')': /simple/mock/
> WARNING: Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x7ffff580ed60>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution')': /simple/mock/
> WARNING: Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x7ffff580ef10>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution')': /simple/mock/
> ERROR: Could not find a version that satisfies the requirement mock (from versions: none)
> ERROR: No matching distribution found for mock
> error: Command '['/nix/store/9xnq0r2a14s9qc5c27ppzq1p7j8wqx22-python3-3.9.13/bin/python3.9', '-m', 'pip', '--disable-pip-version-check', 'wheel', '--no-deps', '-w', '/build/tmp7cwx8pw6', '--quiet', 'mock']' returned non-zero exit status 1.
because
etilsdepends on itself and this causes an infinite recursion. Next I get this error:error: builder for '/nix/store/h76j40mzmc0y2bcy50v4lqfigynn32pd-python3.9-ml_collections-0.1.1.drv' failed with exit code 1; last 10 log lines: > /nix/store/cnnykza37klyikkzb2dwjh1kbhvyhrc9-python3.9-setuptools-61.2.0/lib/python3.9/site-packages/setuptools/installer.py:27: SetuptoolsDeprecationWarning: setuptools.installer is deprecated. Requirements should be satisfied by a PEP 517 installer. > warnings.warn( > WARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x7ffff580e4f0>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution')': /simple/mock/ > WARNING: Retrying (Retry(total=3, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x7ffff580e880>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution')': /simple/mock/ > WARNING: Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x7ffff580eaf0>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution')': /simple/mock/ > WARNING: Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x7ffff580ed60>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution')': /simple/mock/ > WARNING: Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<pip._vendor.urllib3.connection.HTTPSConnection object at 0x7ffff580ef10>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution')': /simple/mock/ > ERROR: Could not find a version that satisfies the requirement mock (from versions: none) > ERROR: No matching distribution found for mock > error: Command '['/nix/store/9xnq0r2a14s9qc5c27ppzq1p7j8wqx22-python3-3.9.13/bin/python3.9', '-m', 'pip', '--disable-pip-version-check', 'wheel', '--no-deps', '-w', '/build/tmp7cwx8pw6', '--quiet', 'mock']' returned non-zero exit status 1.
For this error you can add "mock" to the build inputs of ml-collections (and similarly if it complains about other packages).
For the infinite recusion, maybe you can patch the package to remove the circular dependency?
A bit like this: https://github.com/nix-community/poetry2nix/blob/8b6239cf2ded121f8f3d570d2563d69f05d4208f/overrides/default.nix#L524
Both of these fixes worked! How did you know that mock was a dependency of tensorflow?
Also, I've made some more progress toward getting ml-collections installed but now getting this error:
error: collision between `/nix/store/xcny1kcg647iclpmjss080xzrgj52isv-python3.9-optax-0.1.3/lib/python3.9/site-packages/docs/conf.py' and `/nix/store/7g5p3xfwqq4pv6g1m1f7247pnv2qblv9-python3.9-ml-collections-0.1.1/lib/python3.9/site-packages/docs/conf.py'
flake.nix pyproject.toml poetry.lock
I was able to resolve this by adding
prePatch = ''
export HOME=$TMPDIR;
rm -rf docs/
'';
under ml-collections. I also needed to add doCheck = false under clu. Here is the final working version:
{
inputs = {
nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
utils.url = "github:numtide/flake-utils";
};
outputs = {
self,
nixpkgs,
utils,
}: let
out = system: let
pkgs = import nixpkgs {
inherit system;
};
inherit (pkgs) poetry2nix;
inherit (poetry2nix) mkPoetryEnv;
python = pkgs.python39;
overrides = pyfinal: pyprev: rec {
astunparse = pyprev.astunparse.overridePythonAttrs (old: {
buildInputs = (old.buildInputs or []) ++ [pyfinal.wheel];
});
clu = pyprev.buildPythonPackage rec {
pname = "clu";
version = "0.0.7";
src = pyprev.fetchPypi {
inherit pname version;
sha256 = "sha256-RJqa8XnDpcRPwYlH+4RKAOos0x4+3hMWf/bv6JNn2ys=";
};
doCheck = false;
buildInputs = with pyfinal; [
absl-py
cached-property
etils
flax
jax
jaxlib
ml-collections
numpy
packaging
tensorflow
tensorflow-datasets
];
};
etils = pyprev.etils.overridePythonAttrs (old: {
propagatedBuildInputs =
builtins.filter (i: i.pname != "etils") old.propagatedBuildInputs;
});
# Use cuda-enabled jaxlib as required
jaxlib = pyprev.jaxlibWithCuda.override {
inherit
(pyprev)
absl-py
flatbuffers
numpy
scipy
six
;
};
ml-collections = pyprev.buildPythonPackage rec {
pname = "ml_collections";
version = "0.1.1";
src = pyprev.fetchPypi {
inherit pname version;
sha256 = "sha256-P+/McuxDOqHl0yMHo+R0u7Z/QFvoFOpSohZr/J2+aMw=";
};
buildInputs = with pyfinal; [mock];
propagatedBuildInputs = with pyfinal; [absl-py contextlib2 pyyaml six];
prePatch = ''
export HOME=$TMPDIR;
rm -rf docs/
'';
};
tensorflow =
# Override the nixpkgs bin version instead of
# poetry2nix version so that rpath is set correctly.
pyprev.tensorflow-bin.overridePythonAttrs
{inherit (pyprev.tensorflow) src version;};
};
poetryArgs = {
inherit python;
projectDir = ./.;
preferWheels = true;
overrides = poetry2nix.overrides.withDefaults overrides;
};
poetryEnv = mkPoetryEnv poetryArgs;
in rec {
devShell = pkgs.mkShell rec {
buildInputs = with pkgs; [
poetry
poetryEnv
];
};
};
in
utils.lib.eachDefaultSystem out;
}
with the following pyproject.toml:
[tool.poetry]
name = "ppo"
version = "0.1.0"
description = ""
[tool.poetry.dependencies]
python = ">=3.8,<3.11"
jaxlib = "0.3.0"
jax = "0.3.6"
tensorflow = "^2.9.1"
flax = "0.5.2"
clu = "^0.0.7"
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
and poetry.lock.
I'm happy to put together a pull request of some kind here. Would that be most appropriate for nixpkgs or poetry2nix?