c2rust icon indicating copy to clipboard operation
c2rust copied to clipboard

Published 20 hours ago • verified publisher icon immunant

fatal error: 'algorithm' file not found - `c2rust-ast-exporter` `ExportResult.hpp`

Open milahu opened this issue 2 years ago • 5 comments

bindgen cannot find C++/C header files = https://github.com/rust-lang/rust-bindgen/issues/1834

same problem as https://github.com/immunant/c2rust/issues/323#issuecomment-779224471 and https://github.com/immunant/c2rust/issues/323#issuecomment-788791767

   Compiling c2rust-ast-exporter v0.15.0 (/tmp/c2rust/c2rust-ast-exporter)
warning: `c2rust` (build script) generated 2 warnings
error: failed to run custom build command for `c2rust-ast-exporter v0.15.0 (/tmp/c2rust/c2rust-ast-exporter)`

Caused by:
  process didn't exit successfully: `/tmp/c2rust/target/debug/build/c2rust-ast-exporter-15a5bcc950a5a7b0/build-script-build` (exit status: 1)
  --- stdout
  running: "cmake" "/tmp/c2rust/c2rust-ast-exporter/src" ...
  -- Configuring done
  -- Generating done
  -- Build files have been written to: /tmp/c2rust/target/debug/build/c2rust-ast-exporter-d6a8191d1ceca608/out/build
  running: "cmake" "--build" "." "--target" "clangAstExporter" "--config" "Debug" "--"
  [ 66%] Built target tinycbor_build
  Consolidate compiler generated dependencies of target clangAstExporter
  [100%] Built target clangAstExporter
  ...
  cargo:rustc-link-lib=stdc++

  --- stderr
  src/ExportResult.hpp:11:10: fatal error: 'algorithm' file not found
  src/ExportResult.hpp:11:10: fatal error: 'algorithm' file not found, err: true
  Unable to generate ExportResult bindings

solution concept:

// c2rust/c2rust-ast-exporter/build.rs

    let cppbindings = bindgen::Builder::default()
        .header("src/ExportResult.hpp")
        .whitelist_type("ExportResult")
        .generate_comments(true)
        .derive_default(true)
        // Tell bindgen we are processing c++
        .clang_arg("-xc++")
        .clang_arg("-std=c++11")

        /*
        // fix glibc: warning _FORTIFY_SOURCE requires compiling with optimization (-O)
        .clang_arg("-O")
        */

        // help bindgen to find header files
        // https://github.com/rust-lang/rust-bindgen/issues/1834

        // c2rust/c2rust-ast-exporter/src/ExportResult.hpp:11:10: fatal error: 'algorithm' file not found
        .clang_arg(format!("-I{}/c++/v1", env::var("LIBCXX_INCLUDE_DIR").unwrap()))

        // /nix/store/hzlq2g5man7zi3xr9165gz39ccxxray2-libcxx-11.1.0-dev/include/c++/v1/__config:256:12: fatal error: 'features.h' file not found
        .clang_arg(format!("-I{}", env::var("GLIBC_INCLUDE_DIR").unwrap()))

        // /nix/store/hzlq2g5man7zi3xr9165gz39ccxxray2-libcxx-11.1.0-dev/include/c++/v1/cstddef:44:15: fatal error: 'stddef.h' file not found
        .clang_arg(format!("-I{}", env::var("LIBC_INCLUDE_DIR").unwrap()))

        /*
        // error: could not find native static library `tinycbor`, perhaps an -L flag is missing?
        //.clang_arg(format!("-L{}", env::var("TINYCBOR_LIB_DIR").unwrap()))
        */

        // Finish the builder and generate the bindings.
        .generate()
        .or(Err("Unable to generate ExportResult bindings"))?;

possible alternative: use llvm-config to set these include paths

example env's on nixos:

$ echo $LIBCXX_INCLUDE_DIR
/nix/store/hzlq2g5man7zi3xr9165gz39ccxxray2-libcxx-11.1.0-dev/include

$ echo $GLIBC_INCLUDE_DIR
/nix/store/vccvfa5bjb9dv4x6zq5gjf1yp58y4brg-glibc-2.33-108-dev/include

$ echo $LIBC_INCLUDE_DIR
/nix/store/dxdajaz6l6mzm3iil34qs3rz9cm6gpjz-clang-11.1.0-lib/lib/clang/11.1.0/include

strangely, it works with nix-shell but fails with nix-build (debugging this is on my todo list ...)

c2rust.nix (draft) (WIP) 
/*
nix-build -E 'with import <nixpkgs> {}; callPackage ./c2rust.nix {}'
*/

{ lib, pkgs, fetchFromGitHub, callPackage, makeRustPlatform }:

let
  rustPlatform = mkRustPlatform {
    # https://github.com/immunant/c2rust/blob/master/rust-toolchain
    date = "2021-11-22";
    channel = "nightly";
  };

  mkRustPlatform = { date, channel }:
    let
      mozillaOverlay = fetchFromGitHub {
        owner = "mozilla";
        repo = "nixpkgs-mozilla";
        rev = "cbc7435f5b0b3d17b16fb1d20cf7b616eec5e093";
        sha256 = "LKqAcdL+woWeYajs02bDQ7q8rsqgXuzhC354NoRaV80=";
      };
      mozilla = callPackage "${mozillaOverlay.out}/package-set.nix" {};
      rustSpecific = (mozilla.rustChannelOf { inherit date channel; }).rust;
    in
    makeRustPlatform {
      cargo = rustSpecific;
      rustc = rustSpecific;
    };
in

rustPlatform.buildRustPackage rec {
  pname = "c2rust";
  version = "unstable-2022-04-20";
  src = ./src/c2rust;
  buildType = "debug"; # build faster
  RUST_BACKTRACE = 1; # debug
  LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
  LIBCXX_INCLUDE_DIR = "${pkgs.libcxx.dev}/include";
  GLIBC_INCLUDE_DIR = "${pkgs.glibc.dev}/include";
  LLVM_CONFIG_PATH = "${pkgs.llvm.dev}/bin/llvm-config";
  LLVM_INCLUDE_DIR = "${pkgs.llvm.dev}/include";
  LLVM_LIB_DIR = "${pkgs.llvm.lib}/lib";
  LIBC_INCLUDE_DIR = "${pkgs.libclang.lib}/lib/clang/${pkgs.libclang.version}/include";
  nativeBuildInputs = [
    pkgs.python3 # c2rust-refactor -> python3 process_ast.py
    pkgs.pkgconfig # c2rust-bitfields-derive -> pkg-config --libs --cflags openssl
    pkgs.llvm # c2rust-ast-exporter -> llvm-config
    pkgs.llvm.dev # c2rust-ast-exporter -> #include "llvm/Support/CommandLine.h"
    pkgs.cmake # c2rust-ast-exporter
    pkgs.glibc pkgs.glibc.dev # c2rust-ast-exporter
    pkgs.tinycbor # c2rust-ast-exporter
  ];
  buildInputs = [
    pkgs.openssl
    pkgs.zlib
    pkgs.libclang.lib pkgs.libclang.dev # c2rust-ast-exporter -> #include "clang/Frontend/FrontendActions.h"
    pkgs.libcxx pkgs.libcxx.dev # c2rust-ast-exporter -> #include <algorithm>
    pkgs.tinycbor # c2rust-ast-exporter
  ];
  cargoSha256 = "sha256-y2WwB+JfdNMZR8DHVw4e8CpLhw44LaDvRxxbLkaVUqA=";
  cargoBuildFlags = [
    "--locked"
  ];
  meta = with lib; {
    description = "Migrate C code to Rust";
    homepage = "https://github.com/immunant/c2rust";
    license = licenses.bsd3;
  };
}

todo: add the clang_arg only when the env's are defined

milahu avatar Apr 23 '22 19:04 milahu

@milahu, where would LIBCXX_INCLUDE_DIR, GLIBC_INCLUDE_DIR, and LIBC_INCLUDE_DIR be set? They aren't normally set outside of nix os. Getting these from llvm-config would be better, but I don't think llvm-config has these paths. llvm-config --includedir is for the headers for llvm/clang's own headers, not libc's and libcxx's.

If you make these environment variables optional overrides in c2rust-ast-exporter/build.rs, I thought that would work fine.

kkysen avatar Jun 29 '22 19:06 kkysen

i guess this is mostly a nix problem (multi prefix filesystem)

if c2rust could expose these env-vars this would make the nix side like 1% simpler (less patching)

in nix there is also clangStdenv.mkDerivation instead of the default stdenv.mkDerivation where all these include paths should be set by default

see https://nixos.wiki/wiki/Using_Clang_instead_of_GCC

its just a challenge to combine rustPlatform.buildRustPackage and clangStdenv.mkDerivation maybe something like

rustPlatform.buildRustPackage {
  buildInputs = [
    clangStdenv.cc
  ];
}

what did others do? grep grep grep ...

~/src/nixpkgs $ grep -r buildRustPackage -l | xargs grep -Hni clang 
pkgs/games/blightmud/default.nix:24:  # LIBCLANG_PATH.
pkgs/games/blightmud/default.nix:25:  LIBCLANG_PATH = lib.optionalString withTTS "${llvmPackages.libclang.lib}/lib";
pkgs/games/blightmud/default.nix:30:    # so we have to adapt the BINDGEN_EXTRA_CLANG_ARGS env var to compensate. See
pkgs/games/blightmud/default.nix:35:    export BINDGEN_EXTRA_CLANG_ARGS="$(< ${stdenv.cc}/nix-support/libc-cflags) \
pkgs/games/blightmud/default.nix:37:      -isystem ${llvmPackages.libclang.lib}/lib/clang/${
pkgs/games/blightmud/default.nix:38:        lib.getVersion llvmPackages.clang
pkgs/tools/filesystems/supertag/default.nix:2:, clang, llvmPackages, pkg-config
pkgs/tools/filesystems/supertag/default.nix:23:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/tools/filesystems/supertag/default.nix:25:  nativeBuildInputs = [ clang pkg-config ];
pkgs/tools/misc/vector/default.nix:54:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/tools/misc/tremor-rs/default.nix:33:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/tools/misc/silicon/default.nix:32:  buildInputs = [ llvmPackages.libclang expat freetype fira-code ]
pkgs/tools/misc/silicon/default.nix:39:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/tools/security/fido2luks/default.nix:6:, clang
pkgs/tools/security/fido2luks/default.nix:22:  nativeBuildInputs = [ pkg-config clang ];
pkgs/tools/security/fido2luks/default.nix:25:    export LIBCLANG_PATH="${llvmPackages.libclang.lib}/lib"
pkgs/tools/security/sequoia/default.nix:44:    llvmPackages_12.libclang.lib
pkgs/tools/security/sequoia/default.nix:45:    llvmPackages_12.clang
pkgs/tools/security/sequoia/default.nix:75:  LIBCLANG_PATH = "${llvmPackages_12.libclang.lib}/lib";
pkgs/tools/admin/procs/default.nix:18:  LIBCLANG_PATH = lib.optionals stdenv.isDarwin "${stdenv.cc.cc.lib}/lib/";
pkgs/tools/system/zenith/default.nix:24:  nativeBuildInputs = [ llvmPackages.clang ] ++ lib.optional nvidiaSupport makeWrapper;
pkgs/tools/system/zenith/default.nix:25:  buildInputs = [ llvmPackages.libclang ] ++ lib.optionals stdenv.isDarwin [ IOKit ];
pkgs/tools/system/zenith/default.nix:29:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/tools/networking/innernet/default.nix:28:    clang
pkgs/tools/networking/innernet/default.nix:33:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/tools/networking/bore/default.nix:22:    ++ lib.optional stdenv.isDarwin llvmPackages.libclang;
pkgs/tools/networking/bore/default.nix:29:  LIBCLANG_PATH="${llvmPackages.libclang.lib}/lib";
pkgs/tools/backup/rdedup/default.nix:2:, llvmPackages, clang, xz
pkgs/tools/backup/rdedup/default.nix:18:  nativeBuildInputs = [ pkg-config llvmPackages.libclang clang ];
pkgs/tools/backup/rdedup/default.nix:23:    export LIBCLANG_PATH="${llvmPackages.libclang.lib}/lib"
pkgs/development/interpreters/wasmer/default.nix:39:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/development/tools/parinfer-rust/default.nix:16:  nativeBuildInputs = [ llvmPackages.clang ];
pkgs/development/tools/parinfer-rust/default.nix:17:  buildInputs = [ llvmPackages.libclang ];
pkgs/development/tools/parinfer-rust/default.nix:18:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/development/tools/amazon-qldb-shell/default.nix:3:, clang
pkgs/development/tools/amazon-qldb-shell/default.nix:25:    nativeBuildInputs = [ clang cmake ];
pkgs/development/tools/amazon-qldb-shell/default.nix:26:    buildInputs = [ llvmPackages.libclang ]
pkgs/development/tools/amazon-qldb-shell/default.nix:31:    LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/development/tools/rust/bindgen/unwrapped.nix:1:{ lib, fetchFromGitHub, rustPlatform, clang, rustfmt
pkgs/development/tools/rust/bindgen/unwrapped.nix:23:  buildInputs = [ clang.cc.lib ];
pkgs/development/tools/rust/bindgen/unwrapped.nix:26:    export LIBCLANG_PATH="${clang.cc.lib}/lib"
pkgs/development/tools/rust/bindgen/unwrapped.nix:30:  checkInputs = [ clang ];
pkgs/development/tools/rust/bindgen/unwrapped.nix:39:  passthru = { inherit clang; };
pkgs/development/tools/rust/cargo-spellcheck/default.nix:4:, libclang
pkgs/development/tools/rust/cargo-spellcheck/default.nix:24:  LIBCLANG_PATH = "${libclang.lib}/lib";
pkgs/development/compilers/rust/default.nix:73:      } // lib.optionalAttrs (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) {
pkgs/servers/nosql/influxdb2/default.nix:42:    nativeBuildInputs = [ llvmPackages.libclang ];
pkgs/servers/nosql/influxdb2/default.nix:44:    LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/applications/misc/imag/default.nix:29:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/applications/misc/clipcat/default.nix:2:, pkg-config, llvmPackages, clang, protobuf, python3 }:
pkgs/applications/misc/clipcat/default.nix:17:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/applications/misc/clipcat/default.nix:26:    clang
pkgs/applications/misc/clipcat/default.nix:27:    llvmPackages.libclang
pkgs/applications/blockchains/solana/default.nix:47:  # LIBCLANG_PATH="${llvmPackages.libclang.lib}/lib";
pkgs/applications/blockchains/polkadot/default.nix:1:{ clang
pkgs/applications/blockchains/polkadot/default.nix:39:  nativeBuildInputs = [ clang ];
pkgs/applications/blockchains/polkadot/default.nix:46:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/applications/blockchains/electrs/default.nix:27:  nativeBuildInputs = [ llvmPackages.clang ];
pkgs/applications/blockchains/electrs/default.nix:28:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/applications/blockchains/nearcore/default.nix:48:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/applications/blockchains/nearcore/default.nix:49:  BINDGEN_EXTRA_CLANG_ARGS = "-isystem ${llvmPackages.libclang.lib}/lib/clang/${lib.getVersion llvmPackages.clang}/include";
pkgs/applications/blockchains/snarkos/default.nix:26:  nativeBuildInputs = lib.optionals stdenv.isLinux [ pkg-config llvmPackages.clang ];
pkgs/applications/blockchains/snarkos/default.nix:33:  LIBCLANG_PATH="${llvmPackages.libclang.lib}/lib";
pkgs/applications/networking/remote/rustdesk/default.nix:13:, clang
pkgs/applications/networking/remote/rustdesk/default.nix:42:  LIBCLANG_PATH="${llvmPackages.libclang.lib}/lib";
pkgs/applications/networking/remote/rustdesk/default.nix:63:  nativeBuildInputs = [ pkg-config cmake makeWrapper copyDesktopItems yasm nasm clang wrapGAppsHook ];
pkgs/applications/editors/emacs/elisp-packages/tsc/default.nix:7:, clang
pkgs/applications/editors/emacs/elisp-packages/tsc/default.nix:46:    nativeBuildInputs = [ clang ];
pkgs/applications/editors/emacs/elisp-packages/tsc/default.nix:50:      export LIBCLANG_PATH="${llvmPackages.libclang.lib}/lib"
pkgs/applications/editors/emacs/elisp-packages/tsc/update.py:101:          nativeBuildInputs = [ clang ];
pkgs/applications/editors/vim/plugins/overrides.nix:112:  clang_complete = super.clang_complete.overrideAttrs (old: {
pkgs/applications/editors/vim/plugins/overrides.nix:115:    # These usually implicitly set by cc-wrapper around clang (pkgs/build-support/cc-wrapper).
pkgs/applications/editors/vim/plugins/overrides.nix:116:    # The linked ruby code shows generates the required '.clang_complete' for cmake based projects
pkgs/applications/editors/vim/plugins/overrides.nix:119:      substituteInPlace "$out"/plugin/clang_complete.vim \
pkgs/applications/editors/vim/plugins/overrides.nix:120:        --replace "let g:clang_library_path = '' + "''" + ''" "let g:clang_library_path='${llvmPackages.libclang.lib}/lib/libclang.so'"
pkgs/applications/editors/vim/plugins/overrides.nix:122:      substituteInPlace "$out"/plugin/libclang.py \
pkgs/applications/editors/vim/plugins/overrides.nix:123:        --replace "/usr/lib/clang" "${llvmPackages.clang.cc}/lib/clang"
pkgs/applications/editors/vim/plugins/overrides.nix:129:      sed "/^let g:clighter8_libclang_path/s|')$|${llvmPackages.clang.cc.lib}/lib/libclang.so')|" \
pkgs/applications/editors/vim/plugins/overrides.nix:1141:      "coc-clangd"
pkgs/applications/editors/neovim/neovide/default.nix:63:  LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
pkgs/applications/editors/neovim/neovide/default.nix:77:    llvmPackages.clang # skia
grep: .git/objects/pack/pack-bcc4bea9ba6ef5ee5830f5198ba4b7c9a158b928.pack: binary file matches
doc/languages-frameworks/rust.section.md:468:  `bindgen` find `libclang` and `libclang` find the libraries in `buildInputs`.

for example

pkgs/games/blightmud/default.nix pkgs/development/tools/rust/bindgen/unwrapped.nix

milahu avatar Jun 29 '22 21:06 milahu

~~I'm getting this error building on Ubuntu 22.04 with the nightly toolchain as well.~~

Seems to be rectified after double checking my libstdc++ install, might be something to mention in the readme.

tyler274 avatar Aug 29 '22 11:08 tyler274

For reference, I was experiencing this on my system(s), both Ubuntu 22.04.

The solution has been to install the package libstdc++-12-dev. @kkysen I think this should be added to the README; I'd add this myself, but the details confuse me, specifically, why v12 is required on Ubuntu 22.04, while on Ubuntu 20.04, older versions were fine :grimacing:

64kramsystem avatar Nov 11 '22 18:11 64kramsystem

I can confirm that installing libstdc++-12-dev stopped the compile failures on Ubuntu 22.04, but this seems more like a workaround than a true fix. The compile failures were happening with libstdc++-11-dev, which also provides <algorithm> (and std::fill_n, which seems to be one of the things used by ExportResult.cpp). So it really seems like compiling with libstdc++11-dev ought to work.

abentley avatar May 27 '23 15:05 abentley