How to use MKL with Numpy

[robertodr]

I am trying and failing to create an environment with Numpy linked to MKL. This is my pyproject.toml:

[tool.poetry]
name = "example"
version = "0.1.0"
description = ""
authors = ["Roberto Di Remigio"]

[tool.poetry.dependencies]
python = "^3.7"
numpy = "^1.17"

[build-system]
requires = ["poetry>=0.12"]
build-backend = "poetry.masonry.api"

and this is the shell.nix:

let
  outer = import <nixpkgs> {};
in
  {
    pkgs ? import (outer.fetchFromGitHub {
      # Obtained on 2020-02-16 with `nix-prefetch-github nixos nixpkgs-channels --nix --rev nixpkgs-unstable`
      owner = "nixos";
      repo = "nixpkgs-channels";
      rev = "05f0934825c2a0750d4888c4735f9420c906b388";
      sha256 = "1g8c2w0661qn89ajp44znmwfmghbbiygvdzq0rzlvlpdiz28v6gy";
    }) {}
  }:

with pkgs;
let
  src = fetchFromGitHub {
    owner = "nix-community";
    repo = "poetry2nix";
    rev = "b7b50f4098f811e1952fc158d8ed26250be56b4c";
    sha256 = "0pb7b2ivs5axbq2ayaap9sciyn7zaxzkih6p4l0s53ydmrn5n540";
  };
in
with import "${src.out}/overlay.nix" pkgs pkgs;
let
  pythonEnv = poetry2nix.mkPoetryEnv {
    python = python3;
    poetrylock = ./poetry.lock;
    overrides = [
      poetry2nix.defaultPoetryOverrides
      (self: super: {
        numpy = super.numpy.override {
          blas = pkgs.mkl;
        };
      })
    ];
  };
in
  mkShell {
    name = "example";
    nativeBuildInputs = [
      pythonEnv
      poetry
    ];
  }

Running python -c "import numpy; numpy.show_config() shows that I am linking against OpenBLAS:

blas_mkl_info:
  NOT AVAILABLE
blis_info:
  NOT AVAILABLE
openblas_info:
    libraries = ['openblas', 'openblas']
    library_dirs = ['/nix/store/9iljqxsg4j36h8kl7nwg2iyq34ysglrv-openblas-0.3.8/lib']
    language = c
    define_macros = [('HAVE_CBLAS', None)]
blas_opt_info:
    libraries = ['openblas', 'openblas']
    library_dirs = ['/nix/store/9iljqxsg4j36h8kl7nwg2iyq34ysglrv-openblas-0.3.8/lib']
    language = c
    define_macros = [('HAVE_CBLAS', None)]
lapack_mkl_info:
  NOT AVAILABLE
openblas_lapack_info:
    libraries = ['openblas', 'openblas']
    library_dirs = ['/nix/store/9iljqxsg4j36h8kl7nwg2iyq34ysglrv-openblas-0.3.8/lib']
    language = c
    define_macros = [('HAVE_CBLAS', None)]
lapack_opt_info:
    libraries = ['openblas', 'openblas']
    library_dirs = ['/nix/store/9iljqxsg4j36h8kl7nwg2iyq34ysglrv-openblas-0.3.8/lib']
    language = c
    define_macros = [('HAVE_CBLAS', None)]

Anything obviously wrong I am doing?

[robertodr]

I notice the following during the build phase:

blas_opt_info:
blas_mkl_info:
customize UnixCCompiler
  libraries mkl_rt not found in ['/nix/store/vs4vj1yzqj1bkcqkf3b6sxm6jfy1gb4j-python3-3.7.7/lib', '/usr/local/lib', '/usr/lib', '/usr/lib/']
  NOT AVAILABLE

blis_info:
  libraries blis not found in ['/nix/store/vs4vj1yzqj1bkcqkf3b6sxm6jfy1gb4j-python3-3.7.7/lib', '/usr/local/lib', '/usr/lib', '/usr/lib/']
  NOT AVAILABLE

openblas_info:
C compiler: gcc -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -fPIC

creating /tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmpybhzqvg8/tmp
creating /tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmpybhzqvg8/tmp/nix-build-python3.7-numpy-1.18.2.drv-0
creating /tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmpybhzqvg8/tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmpybhzqvg8
compile options: '-c'
gcc: /tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmpybhzqvg8/source.c
gcc /tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmpybhzqvg8/tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmpybhzqvg8/source.o -L/nix/store/rcgc63zy9mc9vz76p16ylqjsnmadnmrk-openblas-0.3.8/lib -lopenblas -o /tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmpybhzqvg8/a.out
  FOUND:
    libraries = ['openblas', 'openblas']
    library_dirs = ['/nix/store/rcgc63zy9mc9vz76p16ylqjsnmadnmrk-openblas-0.3.8/lib']
    language = c
    define_macros = [('HAVE_CBLAS', None)]

  FOUND:
    libraries = ['openblas', 'openblas']
    library_dirs = ['/nix/store/rcgc63zy9mc9vz76p16ylqjsnmadnmrk-openblas-0.3.8/lib']
    language = c
    define_macros = [('HAVE_CBLAS', None)]

lapack_opt_info:
lapack_mkl_info:
  libraries mkl_rt not found in ['/nix/store/vs4vj1yzqj1bkcqkf3b6sxm6jfy1gb4j-python3-3.7.7/lib', '/usr/local/lib', '/usr/lib', '/usr/lib/']
  NOT AVAILABLE

openblas_lapack_info:
C compiler: gcc -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -fPIC

creating /tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmph8g0ejun/tmp
creating /tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmph8g0ejun/tmp/nix-build-python3.7-numpy-1.18.2.drv-0
creating /tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmph8g0ejun/tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmph8g0ejun
compile options: '-c'
gcc: /tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmph8g0ejun/source.c
gcc /tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmph8g0ejun/tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmph8g0ejun/source.o -L/nix/store/rcgc63zy9mc9vz76p16ylqjsnmadnmrk-openblas-0.3.8/lib -lopenblas -o /tmp/nix-build-python3.7-numpy-1.18.2.drv-0/tmph8g0ejun/a.out
  FOUND:
    libraries = ['openblas', 'openblas']
    library_dirs = ['/nix/store/rcgc63zy9mc9vz76p16ylqjsnmadnmrk-openblas-0.3.8/lib']
    language = c
    define_macros = [('HAVE_CBLAS', None)]

  FOUND:
    libraries = ['openblas', 'openblas']
    library_dirs = ['/nix/store/rcgc63zy9mc9vz76p16ylqjsnmadnmrk-openblas-0.3.8/lib']
    language = c
    define_macros = [('HAVE_CBLAS', None)]

seems the configuration script for Numpy looks for MKL in the wrong folders.

[adisbladis]

I'm hoping to punt on this and let https://github.com/NixOS/nixpkgs/pull/83888 fix it nixpkgs-wide.

[robertodr]

That does look very interesting (and exactly what I'd need right now...), thanks for the link! Any temporary workaround I could use?

[robertodr]

That PR got merged, I'll give it a try and report.

[robertodr]

Still can't make it work. I guess I'm not using it correctly?

let
  outer = import <nixpkgs> {};
in
{ pkgs ? import (
    outer.fetchFromGitHub {
      owner = "nixos";
      repo = "nixpkgs";
      rev = "e5450d9d004141fa9052d72bbb5238e50922b39f";
      sha256 = "175dzv8s26a04czpq12jngkh3xyzk3gsxyb9nn2pp3786sspl582";
    }
  ) {
    overlays = [
      (
        self: super: {
          blas = super.blas.override {
            blasProvider = super.mkl;
          };
        }
      )
    ];
  }
}:

with pkgs;
let
  src = fetchFromGitHub {
    owner = "nix-community";
    repo = "poetry2nix";
    rev = "b16cfe598b199d962bcf05b101662acc26f0c558";
    sha256 = "0rrg3awpp7qzinf71nsg18gjny0y9njbi4vk37h1nl7apms2kh1p";
  };
in
  with import "${src.out}/overlay.nix" pkgs pkgs;
  let
    pythonEnv = poetry2nix.mkPoetryEnv {
      python = python3;
      projectDir = ./.;
    };
  in
  mkShell {
    name = "example";
    nativeBuildInputs = [
      pythonEnv
    ];
  }

[robertodr]

The overlay seems to be completely ignored: MKL is not even downloaded, let alone used. If I use:

nativeBuildInputs = [
   python3.pkgs.numpy
];

things seem to work as expected. Do I need to override the NumPy in some way?

[tbenst]

@robertodr did you succeed in having numpy in a poetry2nix environment with functional MKL? Could you show the full example if so? thanks!!

[robertodr]

No, no luck so far.

On Sun, May 10, 2020, 06:27 Tyler Benster [email protected] wrote:

@robertodr https://github.com/robertodr did you succeed in having numpy in a poetry2nix environment with functional MKL? Could you show the full example if so? thanks!!

— You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub https://github.com/nix-community/poetry2nix/issues/80#issuecomment-626271693, or unsubscribe https://github.com/notifications/unsubscribe-auth/AA4JOENQKGBLN3RVQIOZMQ3RQYUKHANCNFSM4MC7EOFQ .

[robertodr]

@adisbladis I am still not able to make this work, despite NixOS/nixpkgs#83888 (and subsequent PRs on the same topic) having been merged.

[robertodr]

Still no luck here.

[robertodr]

@adisbladis I still cannot get NumPy to work with MKL.

[tscholak]

did it eventually work?

[jimzer]

Did you get it to work?