codebrowser
codebrowser copied to clipboard
fake_compiler works incorrectly on Bazel
I'm trying to create a doc for tensorflow.
First, it's hard to use a dummy gcc on Bazel. (you need to change a lot of things)
And, the simple scripts is not work for Bazel's complicated build command (like
/home/yangff/woboq_codebrowser-master/scripts/gcc -U_FORTIFY_SOURCE '-D_FORTIFY_SOURCE=1' -fstack-protector -Wall -Wl,-z,-relro,-z,now -B/home/yangff/woboq_codebrowser-master/scripts -B/usr/bin -Wunused-but-set-parameter -Wno-free-nonheap-object -fno-canonical-system-headers -fno-omit-frame-pointer -g0 -O2 -DNDEBUG -ffunction-sections -fdata-sections '-std=c++0x' -MD -MF bazel-out/local-py3-opt/bin/tensorflow/core/kernels/_objs/pooling_ops/tensorflow/core/kernels/avgpooling_op.pic.d '-frandom-seed=bazel-out/local-py3-opt/bin/tensorflow/core/kernels/_objs/pooling_ops/tensorflow/core/kernels/avgpooling_op.pic.o' -fPIC -DEIGEN_MPL2_ONLY -iquote . -iquote bazel-out/local-py3-opt/genfiles -iquote external/eigen_archive -iquote bazel-out/local-py3-opt/genfiles/external/eigen_archive -iquote external/bazel_tools -iquote bazel-out/local-py3-opt/genfiles/external/bazel_tools -iquote external/protobuf -iquote bazel-out/local-py3-opt/genfiles/external/protobuf -iquote external/com_googlesource_code_re2 -iquote bazel-out/local-py3-opt/genfiles/external/com_googlesource_code_re2 -iquote external/farmhash_archive -iquote bazel-out/local-py3-opt/genfiles/external/farmhash_archive -iquote external/gif_archive -iquote bazel-out/local-py3-opt/genfiles/external/gif_archive -iquote external/highwayhash -iquote bazel-out/local-py3-opt/genfiles/external/highwayhash -iquote external/jpeg_archive -iquote bazel-out/local-py3-opt/genfiles/external/jpeg_archive -iquote external/png_archive -iquote bazel-out/local-py3-opt/genfiles/external/png_archive -iquote external/zlib_archive -iquote bazel-out/local-py3-opt/genfiles/external/zlib_archive -isystem external/eigen_archive -isystem bazel-out/local-py3-opt/genfiles/external/eigen_archive -isystem external/bazel_tools/tools/cpp/gcc3 -isystem external/protobuf/src -isystem bazel-out/local-py3-opt/genfiles/external/protobuf/src -isystem external/farmhash_archive -isystem bazel-out/local-py3-opt/genfiles/external/farmhash_archive -isystem external/gif_archive -isystem bazel-out/local-py3-opt/genfiles/external/gif_archive -isystem external/highwayhash -isystem bazel-out/local-py3-opt/genfiles/external/highwayhash -isystem external/jpeg_archive -isystem bazel-out/local-py3-opt/genfiles/external/jpeg_archive -isystem external/png_archive -isystem bazel-out/local-py3-opt/genfiles/external/png_archive -isystem external/zlib_archive -isystem bazel-out/local-py3-opt/genfiles/external/zlib_archive -fno-exceptions -DEIGEN_AVOID_STL_ARRAY -pthread -fno-canonical-system-headers -Wno-builtin-macro-redefined '-D__DATE__="redacted"' '-D__TIMESTAMP__="redacted"' '-D__TIME__="redacted"' -c tensorflow/core/kernels/avgpooling_op.cc -o bazel-out/local-py3-opt/bin/tensorflow/core/kernels/_objs/pooling_ops/tensorflow/core/kernels/avgpooling_op.pic.o)
)
I've a idea for it, use --subcommands
and recover compile info from it. I'm trying this and if I success I will post it here.
var fs = require('fs')
function replaceAll(target, search, replacement) {
return target.replace(new RegExp(search, 'g'), replacement);
};
var txt = replaceAll(replaceAll(fs.readFileSync('command.log').toString(), "\r\n", ""), "\n", "")
var regex_act = /\[action 'Compiling ([^']*).*?(?=gcc )gcc ([^\)]*)/g
var objs = []
var basepath = '/home/yangff/.cache/bazel/_bazel_yangff/a35e3a3265ab567ea3bbc173a204d1ed/execroot/tensorflow-master/'
var cnt = 0
do {
m = regex_act.exec(txt)
if (m) {
console.log(m[1])
cnt = cnt+ 1
if (m[1].endsWith(' [for host]'))
m[1] = m[1].substring(0, m[1].length - 11)
objs.push({
directory: basepath,
command: 'c++ ' + replaceAll(m[2], "-fno-canonical-system-headers", ""),
file: basepath + m[1]
})
}
} while (m);
console.log(cnt)
fs.writeFileSync('compile_commands.json', JSON.stringify(objs))
I have no experience with Bazel.
Googling a bit i found some other people wanting to generate the compile_commands: for example https://gist.github.com/bsilver8192/0115ee5d040bb601e3b7 and https://github.com/cppit/jucipp/issues/250
It would be nice if bazel could support that
However, it might be easier to do the build without compilation_command.json
Instead of passing the -b
argument to codebrowser_generator
, you can pass the compiler arguments in the command line after --
So you can use simple something like this:
codebrowser_generator path/to/tensorflow -o path/to/html -- -U_FORTIFY_SOURCE '-D_FORTIFY_SOURCE=1' -fstack-protector -Wall -Wl,-z,-relro,-z,now -B/home/yangff/woboq_codebrowser-master/scripts -B/usr/bin -Wunused-but-set-parameter -Wno-free-nonheap-object -fno-canonical-system-headers -fno-omit-frame-pointer -g0 -O2 -DNDEBUG -ffunction-sections -fdata-sections '-std=c++0x' -MD -MF bazel-out/local-py3-opt/bin/tensorflow/core/kernels/_objs/pooling_ops/tensorflow/core/kernels/avgpooling_op.pic.d '-frandom-seed=bazel-out/local-py3-opt/bin/tensorflow/core/kernels/_objs/pooling_ops/tensorflow/core/kernels/avgpooling_op.pic.o' -fPIC -DEIGEN_MPL2_ONLY -iquote . -iquote bazel-out/local-py3-opt/genfiles -iquote external/eigen_archive -iquote bazel-out/local-py3-opt/genfiles/external/eigen_archive -iquote external/bazel_tools -iquote bazel-out/local-py3-opt/genfiles/external/bazel_tools -iquote external/protobuf -iquote bazel-out/local-py3-opt/genfiles/external/protobuf -iquote external/com_googlesource_code_re2 -iquote bazel-out/local-py3-opt/genfiles/external/com_googlesource_code_re2 -iquote external/farmhash_archive -iquote bazel-out/local-py3-opt/genfiles/external/farmhash_archive -iquote external/gif_archive -iquote bazel-out/local-py3-opt/genfiles/external/gif_archive -iquote external/highwayhash -iquote bazel-out/local-py3-opt/genfiles/external/highwayhash -iquote external/jpeg_archive -iquote bazel-out/local-py3-opt/genfiles/external/jpeg_archive -iquote external/png_archive -iquote bazel-out/local-py3-opt/genfiles/external/png_archive -iquote external/zlib_archive -iquote bazel-out/local-py3-opt/genfiles/external/zlib_archive -isystem external/eigen_archive -isystem bazel-out/local-py3-opt/genfiles/external/eigen_archive -isystem external/bazel_tools/tools/cpp/gcc3 -isystem external/protobuf/src -isystem bazel-out/local-py3-opt/genfiles/external/protobuf/src -isystem external/farmhash_archive -isystem bazel-out/local-py3-opt/genfiles/external/farmhash_archive -isystem external/gif_archive -isystem bazel-out/local-py3-opt/genfiles/external/gif_archive -isystem external/highwayhash -isystem bazel-out/local-py3-opt/genfiles/external/highwayhash -isystem external/jpeg_archive -isystem bazel-out/local-py3-opt/genfiles/external/jpeg_archive -isystem external/png_archive -isystem bazel-out/local-py3-opt/genfiles/external/png_archive -isystem external/zlib_archive -isystem bazel-out/local-py3-opt/genfiles/external/zlib_archive -fno-exceptions -DEIGEN_AVOID_STL_ARRAY -pthread -fno-canonical-system-headers -Wno-builtin-macro-redefined '-D__DATE__="redacted"' '-D__TIMESTAMP__="redacted"' '-D__TIME__="redacted"'
Hi @ogoffart ,
I am wondering how did you get that very long line of compiler arguments for Tensorflow?
Thanks! Yi
@wangkuiyi
I'm using bazel's verbose logs (with --subcommands
) and that script run bazel with --experimental_action_listener=//tools/actions:generate_compile_commands_listener
Hi @Yangff , Thank you for the hint and the awesome work at http://yangff.coding.me/tf-doc/codebrowser/codebrowser/tensorflow !
But when I tried to follow your hint, I got an error complaining that the rule //tools/actions:generate_compiler_commands_listener
couldn't be found. And I checked that there isn't a directory actions
in ~/work/tensorflow/tools/
.
yi@WangYis-iMac:~/work/tensorflow (r1.1)*$ bazel build --config=opt --experimental_action_listener=//tools/actions:generate_compile_commands_listener //tensorflow/tools/pip_package:build_pip_package
ERROR: /Users/yi/work/tensorflow/third_party/mkl/BUILD:3:1: every rule of type config_setting implicitly depends upon the target '//tools/actions:generate_compile_commands_listener', but this target could not be found because of: no such package 'tools/actions': BUILD file not found on package path.
ERROR: Analysis of target '//tensorflow/tools/pip_package:build_pip_package' failed; build aborted.
INFO: Elapsed time: 5.339s
If this is still an issue, you may also try out https://github.com/grailbio/bazel-compilation-database as an alternative.