poetry2nix
poetry2nix copied to clipboard
Published
20 hours ago •
nix-community
nix-community
Tokenizers does not build using wheel
Describe the issue
Even though tokenizers is set to build using a wheel with preferWheel = true, nix attempts to build it directly.
Note that there are other issues building this package, it probably won't build correctly after this is fixed.
Additional context
build log:
Sourcing python-remove-tests-dir-hook
Sourcing python-catch-conflicts-hook.sh
Sourcing python-remove-bin-bytecode-hook.sh
Sourcing pip-install-hook
Using pipInstallPhase
Sourcing python-imports-check-hook.sh
Using pythonImportsCheckPhase
Sourcing python-namespaces-hook
Sourcing pip-build-hook
Using pipBuildPhase
Using pipShellHook
@nix { "action": "setPhase", "phase": "unpackPhase" }
unpacking sources
unpacking source archive /nix/store/9kwanmma9mga37kxva4cn5za55xmd3s1-tokenizers-0.13.2.tar.gz
source root is tokenizers-0.13.2
setting SOURCE_DATE_EPOCH to timestamp 1667808203 of file tokenizers-0.13.2/setup.cfg
@nix { "action": "setPhase", "phase": "patchPhase" }
patching sources
Removing path dependencies
Finished removing path dependencies
Removing git dependencies
Finished removing git dependencies
@nix { "action": "setPhase", "phase": "configurePhase" }
configuring
no configure script, doing nothing
@nix { "action": "setPhase", "phase": "buildPhase" }
building
Executing pipBuildPhase
Creating a wheel...
[33mWARNING: The directory '/homeless-shelter/.cache/pip' or its parent directory is not owned or is not writable by the current user. The cache has been disabled. Check the permissions and owner of that directory. If executing pip with sudo, you should use sudo's -H flag.[0m[33m
[0mProcessing /build/tokenizers-0.13.2
Running command Preparing metadata (pyproject.toml)
Traceback (most recent call last):
File "/nix/store/44gzm67z86lhd7i3gzyxm1rwyb6j451d-python3.9-pip-22.2.2/lib/python3.9/site-packages/pip/_vendor/pep517/in_process/_in_process.py", line 363, in <module>
main()
File "/nix/store/44gzm67z86lhd7i3gzyxm1rwyb6j451d-python3.9-pip-22.2.2/lib/python3.9/site-packages/pip/_vendor/pep517/in_process/_in_process.py", line 345, in main
json_out['return_val'] = hook(**hook_input['kwargs'])
File "/nix/store/44gzm67z86lhd7i3gzyxm1rwyb6j451d-python3.9-pip-22.2.2/lib/python3.9/site-packages/pip/_vendor/pep517/in_process/_in_process.py", line 164, in prepare_metadata_for_build_wheel
return hook(metadata_directory, config_settings)
File "/nix/store/9lwsv9w6mzgb8h9sd4c114z45isp4bfp-python3.9-setuptools-65.3.0/lib/python3.9/site-packages/setuptools/build_meta.py", line 377, in prepare_metadata_for_build_wheel
self.run_setup()
File "/nix/store/9lwsv9w6mzgb8h9sd4c114z45isp4bfp-python3.9-setuptools-65.3.0/lib/python3.9/site-packages/setuptools/build_meta.py", line 335, in run_setup
exec(code, locals())
File "<string>", line 2, in <module>
ModuleNotFoundError: No module named 'setuptools_rust'
[1;31merror[0m: [1msubprocess-exited-with-error[0m
[31m×[0m [32mPreparing metadata [0m[1;32m([0m[32mpyproject.toml[0m[1;32m)[0m did not run successfully.
[31m│[0m exit code: [1;36m1[0m
[31m╰─>[0m See above for output.
[1;35mnote[0m: This error originates from a subprocess, and is likely not a problem with pip.
[1;35mfull command[0m: [34m/nix/store/ps37lailxq0xwkhjvvrwrwkw42xi7v3k-python3-3.9.16/bin/python3.9 /nix/store/44gzm67z86lhd7i3gzyxm1rwyb6j451d-python3.9-pip-22.2.2/lib/python3.9/site-packages/pip/_vendor/pep517/in_process/_in_process.py prepare_metadata_for_build_wheel /build/tmpvbijwxhj[0m
[1;35mcwd[0m: /build/tokenizers-0.13.2
Preparing metadata (pyproject.toml) ... [?25l[?25herror
[1;31merror[0m: [1mmetadata-generation-failed[0m
[31m×[0m Encountered error while generating package metadata.
[31m╰─>[0m See above for output.
[1;35mnote[0m: This is an issue with the package mentioned above, not pip.
[1;36mhint[0m: See above for details.
flake.nix:
{
description = "Application packaged using poetry2nix";
inputs.flake-utils.url = "github:numtide/flake-utils";
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
inputs.poetry2nix = {
url = "github:nix-community/poetry2nix";
inputs.nixpkgs.follows = "nixpkgs";
};
outputs = { self, nixpkgs, flake-utils, poetry2nix }:
flake-utils.lib.eachDefaultSystem (system:
let
# see https://github.com/nix-community/poetry2nix/tree/master#api for more functions and examples.
inherit (poetry2nix.legacyPackages.${system}) mkPoetryApplication;
pkgs = nixpkgs.legacyPackages.${system};
poetry = poetry2nix.legacyPackages.${system};
in
{
packages = {
myapp = mkPoetryApplication {
name = "mokuro-wrapped";
projectDir = ./.;
python = pkgs.python39;
overrides = poetry.defaultPoetryOverrides.extend
(self: super: {
torchsummary = super.torchsummary.overridePythonAttrs (
old: {
buildInputs = (old.buildInputs or []) ++ [ super.setuptools ];
});
# inlined
# fugashi = (import ./fugashi.nix inputs);
fugashi =
with pkgs; with python39.pkgs;
buildPythonPackage rec {
pname = "fugashi";
version = "1.2.1";
src = fetchPypi {
inherit pname version;
sha256 = "58eupmhABq8iO1FFuWwHHC4QNTr7nfZxLfRFgAPPYtM=";
};
postPatch = ''
substituteInPlace setup.py \
--replace 'sys.argv[1] == "sdist"' 'True' \
--replace 'output, data_files = [], []' \
'output, data_files = ["${mecab}/include", "${mecab}/lib", "mecab stdc++"], []'
'';
nativeBuildInputs = [
cython
setuptools-scm
];
propagatedBuildInputs = [
#mecab-python3
#unidic-lite
];
};
});
};
default = self.packages.${system}.myapp;
};
devShells.default = pkgs.mkShell {
packages = [ poetry2nix.packages.${system}.poetry ];
};
});
}
flake.lock
{
"nodes": {
"flake-utils": {
"locked": {
"lastModified": 1667395993,
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_2": {
"locked": {
"lastModified": 1610051610,
"narHash": "sha256-U9rPz/usA1/Aohhk7Cmc2gBrEEKRzcW4nwPWMPwja4Y=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "3982c9903e93927c2164caa727cd3f6a0e6d14cc",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1674641431,
"narHash": "sha256-qfo19qVZBP4qn5M5gXc/h1MDgAtPA5VxJm9s8RUAkVk=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "9b97ad7b4330aacda9b2343396eb3df8a853b4fc",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"poetry2nix": {
"inputs": {
"flake-utils": "flake-utils_2",
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1674727911,
"narHash": "sha256-5yIiStrXn/YQbuPY24a0H9ZcjuLRMCRGEmImJRr7Ons=",
"owner": "nix-community",
"repo": "poetry2nix",
"rev": "ea63bd8d4ae6d609cb5cf385e78019213792aa33",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "poetry2nix",
"type": "github"
}
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs",
"poetry2nix": "poetry2nix"
}
}
},
"root": "root",
"version": 7
}
pyproject.toml:
[tool.poetry]
name = "mokuro_wrapper"
version = "0.1.0"
description = ""
authors = []
[tool.poetry.dependencies]
python = "^3.9"
mokuro = "^0.1.6"
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
[tool.poetry.scripts]
mokuro_wrapper = "mokuro_wrapper:main"
poetry.lock: https://pastebin.com/cQCRE850 (too long for issue body)
I also just ran into this issue. tokenizers -> torchhub -> transformers -> openai-whisper. I'm guessing poetry2nix currently cannot be used if you want to use openai-whisper.
It specifically is missing setuptools_rust and not setuptools.
I got a bit further by using this override:
tokenizers = super.tokenizers.overridePythonAttrs (oldAttrs: {
nativeBuildInputs = (oldAttrs.nativeBuildInputs or [])
++ (with pkgs.rustPlatform; [
rust.rustc
rust.cargo
self.setuptools-rust
]);
});
The next problem is that tokenizers doesn't have a Cargo.lock, so it'll fail trying to find it. I created an issue for this: https://github.com/huggingface/tokenizers/issues/1226
Lock files are now part of tokenizers releases. See https://github.com/huggingface/tokenizers/issues/1226#issuecomment-1875072477