node2nix
node2nix copied to clipboard
svanderburg
nix-build fails to build binaries: getaddrinfo ENOTFOUND raw.githubusercontent.com raw.githubusercontent.com:443
Referencing Issue 68 as it seems similar.
I am getting a number of packages, all binaries it seems, failing with the following error after having run node2nix --development and then nix-build -A package: RequestError: getaddrinfo ENOTFOUND github.com github.com:443`
I have also tried a few variants of the following, entering into a shell: nix-shell -p pngquant optipng mozjpeg gifsicle libpng.dev pkgconfig stdenv
I'm loving NixOS, but am struggling with the Node workflow.
> [email protected] postinstall /nix/store/s4mxq1a5kfg7w7x1zj9g5x2wbf3z0i9s-node-protea-api-server-0.2.0/lib/node_modules/protea-api-server/node_modules/gifsicle
> node lib/install.js
⚠ getaddrinfo ENOTFOUND raw.githubusercontent.com raw.githubusercontent.com:443
⚠ gifsicle pre-build test failed
ℹ compiling from source
✖ RequestError: getaddrinfo ENOTFOUND github.com github.com:443
at ClientRequest.<anonymous> (/nix/store/s4mxq1a5kfg7w7x1zj9g5x2wbf3z0i9s-node-protea-api-server-0.2.0/lib/node_modules/protea-api-server/node_modules/got/index.js:74:21)
at ClientRequest.g (events.js:292:16)
at emitOne (events.js:96:13)
at ClientRequest.emit (events.js:188:7)
at TLSSocket.socketErrorListener (_http_client.js:314:9)
at emitOne (events.js:96:13)
at TLSSocket.emit (events.js:188:7)
at connectErrorNT (net.js:1034:8)
at _combinedTickCallback (internal/process/next_tick.js:80:11)
at process._tickCallback (internal/process/next_tick.js:104:9)
> [email protected] postinstall /nix/store/s4mxq1a5kfg7w7x1zj9g5x2wbf3z0i9s-node-protea-api-server-0.2.0/lib/node_modules/protea-api-server/node_modules/mozjpeg
> node lib/install.js
⚠ getaddrinfo ENOTFOUND raw.githubusercontent.com raw.githubusercontent.com:443
⚠ mozjpeg pre-build test failed
ℹ compiling from source
✖ RequestError: getaddrinfo ENOTFOUND github.com github.com:443
at ClientRequest.<anonymous> (/nix/store/s4mxq1a5kfg7w7x1zj9g5x2wbf3z0i9s-node-protea-api-server-0.2.0/lib/node_modules/protea-api-server/node_modules/got/index.js:74:21)
at ClientRequest.g (events.js:292:16)
at emitOne (events.js:96:13)
at ClientRequest.emit (events.js:188:7)
at TLSSocket.socketErrorListener (_http_client.js:314:9)
at emitOne (events.js:96:13)
at TLSSocket.emit (events.js:188:7)
at connectErrorNT (net.js:1034:8)
at _combinedTickCallback (internal/process/next_tick.js:80:11)
at process._tickCallback (internal/process/next_tick.js:104:9)
> [email protected] postinstall /nix/store/s4mxq1a5kfg7w7x1zj9g5x2wbf3z0i9s-node-protea-api-server-0.2.0/lib/node_modules/protea-api-server/node_modules/optipng-bin
> node lib/install.js
⚠ getaddrinfo ENOTFOUND raw.githubusercontent.com raw.githubusercontent.com:443
⚠ optipng pre-build test failed
ℹ compiling from source
✖ RequestError: getaddrinfo ENOTFOUND downloads.sourceforge.net downloads.sourceforge.net:443
at ClientRequest.<anonymous> (/nix/store/s4mxq1a5kfg7w7x1zj9g5x2wbf3z0i9s-node-protea-api-server-0.2.0/lib/node_modules/protea-api-server/node_modules/got/index.js:74:21)
at ClientRequest.g (events.js:292:16)
at emitOne (events.js:96:13)
at ClientRequest.emit (events.js:188:7)
at TLSSocket.socketErrorListener (_http_client.js:314:9)
at emitOne (events.js:96:13)
at TLSSocket.emit (events.js:188:7)
at connectErrorNT (net.js:1034:8)
at _combinedTickCallback (internal/process/next_tick.js:80:11)
at process._tickCallback (internal/process/next_tick.js:104:9)
> [email protected] postinstall /nix/store/s4mxq1a5kfg7w7x1zj9g5x2wbf3z0i9s-node-protea-api-server-0.2.0/lib/node_modules/protea-api-server/node_modules/pngquant-bin
> node lib/install.js
⚠ getaddrinfo ENOTFOUND raw.github.com raw.github.com:443
⚠ pngquant pre-build test failed
ℹ compiling from source
✔ pngquant pre-build test passed successfully
✖ RequestError: pngquant failed to build, make sure that libpng-dev is installed
at StdError (/nix/store/s4mxq1a5kfg7w7x1zj9g5x2wbf3z0i9s-node-protea-api-server-0.2.0/lib/node_modules/protea-api-server/node_modules/pngquant-bin/node_modules/got/index.js:410:3)
at /nix/store/s4mxq1a5kfg7w7x1zj9g5x2wbf3z0i9s-node-protea-api-server-0.2.0/lib/node_modules/protea-api-server/node_modules/pngquant-bin/node_modules/got/index.js:430:3
at ClientRequest.req.once.err (/nix/store/s4mxq1a5kfg7w7x1zj9g5x2wbf3z0i9s-node-protea-api-server-0.2.0/lib/node_modules/protea-api-server/node_modules/pngquant-bin/node_modules/got/index.js:111:21)
at ClientRequest.g (events.js:292:16)
at emitOne (events.js:96:13)
at ClientRequest.emit (events.js:188:7)
at Socket.socketErrorListener (_http_client.js:314:9)
at emitOne (events.js:96:13)
at Socket.emit (events.js:188:7)
at connectErrorNT (net.js:1034:8)
npm ERR! Linux 4.14.84
npm ERR! argv "/nix/store/6yamyrckc6p984zsmikkp0qyis82p6ar-nodejs-6.14.4/bin/node" "/nix/store/6yamyrckc6p984zsmikkp0qyis82p6ar-nodejs-6.14.4/bin/npm" "--registry" "http://www.example.com" "--nodedir=/nix/store/kz4slv82xsbwhrpn2sl504c0csw4hnrn-node-sources" "rebuild"
npm ERR! node v6.14.4
npm ERR! npm v3.10.10
npm ERR! code ELIFECYCLE
npm ERR! [email protected] postinstall: `node lib/install.js`
npm ERR! Exit status 1
npm ERR!
npm ERR! Failed at the [email protected] postinstall script 'node lib/install.js'.
npm ERR! Make sure you have the latest version of node.js and npm installed.
npm ERR! If you do, this is most likely a problem with the pngquant-bin package,
npm ERR! not with npm itself.
npm ERR! Tell the author that this fails on your system:
npm ERR! node lib/install.js
npm ERR! You can get information on how to open an issue for this project with:
npm ERR! npm bugs pngquant-bin
npm ERR! Or if that isn't available, you can get their info via:
npm ERR! npm owner ls pngquant-bin
npm ERR! There is likely additional logging output above.
npm ERR! Please include the following file with any support request:
npm ERR! /nix/store/s4mxq1a5kfg7w7x1zj9g5x2wbf3z0i9s-node-protea-api-server-0.2.0/lib/node_modules/protea-api-server/npm-debug.log
builder for '/nix/store/lx8ilx9ygh46hd1ajamifljkhr77wxnr-node-protea-api-server-0.2.0.drv' failed with exit code 1
error: build of '/nix/store/lx8ilx9ygh46hd1ajamifljkhr77wxnr-node-protea-api-server-0.2.0.drv' failed
Using npm to install the packages globally, after having used npm config set prefix, and with the following dependencies nodejs-10_x pkgconfig autoconf automake libtool nasm autogen zlib libpng results in a successful build for all but the libpng-bin package. Despite this, nix-build still can't build those packages from the node2nix config.
My guess is I should be able to override the following script for gifsicle in some sort of Nix configuration, which nix-build then uses.
'use strict';
1 var BinBuild = require('bin-build');
2 var log = require('logalot');
3 var bin = require('./');
4
5 bin.run(['--version'], function (err) {
6 if (err) {
7 log.warn(err.message);
8 log.warn('gifsicle pre-build test failed');
9 log.info('compiling from source');
10
11 var cfg = [
12 './configure --disable-gifview --disable-gifdiff',
13 '--prefix="' + bin.dest() + '" --bindir="' + bin.dest() + '"'
14 ].join(' ');
15
16 var builder = new BinBuild()
17 .src('https://github.com/kohler/gifsicle/archive/v1.88.tar.gz')
18 .cmd('autoreconf -ivf')
19 .cmd(cfg)
20 .cmd('make install');
21
22 return builder.run(function (err) {
23 if (err) {
24 log.error(err.stack);
25 return;
26 }
27
28 log.success('gifsicle built successfully');
29 });
30 }
31
32 log.success('gifsicle pre-build test passed successfully');
33 });
Ideally though, the previous step should not have failed, and the following binary should have been downloaded: https://raw.githubusercontent.com/imagemin/gifsicle-bin/v3.0.4/vendor/linux/x64/gifsicle
I do understand the Nix packages being immutable though, but somehow need to get that binary into the Node project environment.
Copying the gifsicle store folder to my home directory and running npm install finishes successfully:
> [email protected] postinstall /home/bscholtz/DUMP/gifsicle
> node lib/install.js
⚠ spawn /home/bscholtz/DUMP/gifsicle/vendor/gifsicle ENOENT
⚠ gifsicle pre-build test failed
ℹ compiling from source
✔ gifsicle built successfully
npm notice created a lockfile as package-lock.json. You should commit this file.
added 769 packages from 568 contributors and audited 4493 packages in 30.652s
found 10 vulnerabilities (6 low, 2 moderate, 1 high, 1 critical)
run `npm audit fix` to fix them, or `npm audit` for details
Reference to SO answer to question I posed: https://unix.stackexchange.com/a/487385/255720
@BenSchZA The problem is that the gifsicle package has a postinstall script (install.js) attempting to download external packages. Unfortunately, downloading undeclared dependencies in a Nix build environment is typically restricted.
In single user Nix installations (or Nix installations without sandboxing), the build may still succeed because there is no network access restriction, but in sandboxed Nix installations (such as Nix that comes with NixOS) network access is restricted in a build. This means builds can never download anything from remote locations.
Only fixed output derivations (e.g. Nix functions such as fetchurl and fetchgit) can download external artifacts because their output hashes must be provided in advance, so that Nix can check that they have the expected results. If Nix would allow undeclared downloads, then it is no longer possible to predict the Nix store hash in advance and build are no longer pure.
If you want to deploy these packages with Nix, then you must use the Nix expression language to override the generated NPM expressions. In the override expression, you must download the gifiscle dependency yourself and use the preRebuild hook to modify the install script so that it uses that version. Then it should no longer download it itself.
This is probably very inconvenient for Nix beginners because there is no way to auto generate such expressions.
If you want to investigate this, read the README.md file section about the preRebuild hook and overrides. You can create your own fetchurl function invocation to download the dependency.
Thanks @svanderburg . I appreciate everything you've said about the reason for the errors - that certainly helps me understand the problem. The preRebuild hook and overriding the package is going a bit over my head at the moment. I've read the README and it's a very specific use case - I'm struggling a bit to set it up properly.
I came across someone asking a similar question which you answered here: https://github.com/svanderburg/node2nix/issues/84
This is the WIP override.nix (it's mid edit, and I've been fiddling with nix repl default.nix to access the right attributes etc.):
{pkgs ? import <nixpkgs> {
inherit system;
}, system ? builtins.currentSystem, fetchurl ? []}:
let
nodePackages = import ./default.nix {
inherit pkgs system;
};
in
nodePackages // {
gifsicle = nodePackages."gifsicle-3.0.4".override {
/* name = "gifsicle";
packageName = "gifsicle";
version = "3.0.4"; */
src = fetchurl {
/* url = "https://github.com/kohler/gifsicle/archive/v1.88.tar.gz";
sha256 = "0ajgnjwl2a9k5kx063cg2wfvz1c2infjwy4145zqrgvk6fg3hp9a"; */
url = "https://raw.githubusercontent.com/imagemin/gifsicle-bin/v${3.0.4}/vendor/linux/x64/gifsicle";
sha256 = "1bv156sywnz0z6xhi41ybbwn2g5g159xsadf5gw38ixb48ciwbfg";
};
dontNpmInstall = true;
/* doCheck = true;
checkPhase = ''
./$out/vendor/gifsicle --version
''; */
preRebuild = ''
cp src/gifsicle $out/vendor
'';
postInstall = ''
'';
};
}
What I'm attempting to do is download the binary, using the link extracted from the postinstall script for the package, and include it in the package. My confusion comes in because I have the package gifsicle that fails to install, I'm now using the preRebuild hook to download the binary and do what the postinstall script was trying to do, but do I now stop the postinstall script from running so it doesn't throw the error, or create a new package with just the binary, or does the override do that... I think this is going to take a lot of fiddling, and I can't find any good examples or documentation of how to do this in the context of node2nix.
I appreciate the guidance!
I can post working example for gifsicle build though I'm using different version so some details might need to be changed.
first I don't know how to do it in case gifsicle is a dependecy defined in package.json as described in https://github.com/svanderburg/node2nix/issues/148 so I'm using packages.json file defined as:
$ cat packages.json
[ { "gifsicle": "4.0.1" } ]
Then I used node2nix to generate some expressions for me:
$ node2nix --input packages.json --output node-packages.nix --composition composition.nix --node-env node-env.nix
and finally I had to define overrides in default.nix:
$ cat default.nix
let
pkgs = import <nixpkgs> {};
in
(import ./composition.nix { inherit pkgs; })."gifsicle-4.0.1".override {
dontNpmInstall = true;
preRebuild = ''
ln -sf ${pkgs.gifsicle}/bin/gifsicle vendor/gifsicle
ln -sf ${pkgs.gifsicle}/bin/gifdiff vendor/gifdiff
'';
}
This should build:
$ nix-build
.....
/nix/store/ax7yy7fz55hlps1gffin2phf0877347r-node_gifsicle-4.0.1
I never actually tried the lib though but I hope it will work.
Thanks @turboMaCk! I'll give your solution a try. My work around up until now was using an Ubuntu nspawn container... which wasn't a terrible solution, but definitely not Nix style.
@turboMaCk
using your exact configuration I get the following error on nix-build:
error: attribute 'gifsicle-4.0.1' missing, at /my-project-path/override.nix:27:3
The gifsicle entry in the node-packages.nix looks like this:
let
sources = {
# ...
"gifsicle-4.0.1" = {
name = "gifsicle";
packageName = "gifsicle";
version = "4.0.1";
src = fetchurl {
url = "https://registry.npmjs.org/gifsicle/-/gifsicle-4.0.1.tgz";
sha512 = "A/kiCLfDdV+ERV/UB+2O41mifd+RxH8jlRG8DMxZO84Bma/Fw0htqZ+hY2iaalLRNyUu7tYZQslqUBJxBggxbg==";
};
};
# ...
};
args = {
# ...
};
in
{
tarball = nodeEnv.buildNodeSourceDist args;
package = nodeEnv.buildNodePackage args;
shell = nodeEnv.buildNodeShell args;
}
@FelschR you should be importing composition not node-packages. I'm not sure what might have change with updates of node2nix though first thing to look at is if you've generated and imported generated files the same way as I did.
its got fixed . I have downloaded gifsicle,mozjpeg,optipng on my local system and than copied all the items in node_modules folder on main server. You need to copy its .bin files as well from node_modules folder.
if its doen't work then also download imagemin,imagemin-gifsicle,imagemin-mozjpeg,imagemin-optipng and imagemin-svgo modules and follow the same as above.