Support cross-compilation for arm64

[katre]

Instead of requiring builds take place on arm64 hardware, we should try to cross-compile from the current CentOS builder.

There are a few steps needed:

1. Make sure the right tools are in place in the gcr.io/bazel-public/centos7-releaser:latest image
2. Define the cc_toolchain entries for the tools, that support cross-compilation
3. Define the cc_toolchain_suite (legacy) or toolchain and platform (new hotness) targets needed to support cross-compilation.
   1. This may also mean further changes to bazel to support enabling --incompatible_enable_cc_toolchain_resolution for Bazel itself
4. Update the release scripts to properly use the new config.

[katre]

@philwo As a starting point, could you please capture the output of running bazel query @local_config_cc//:all --output=build on the current arm build machine?

[katre]

Once this is in, we could explore cross-compiling other architectures, although darwin and windows builds are trickier to manage from a linux.

[philwo]

This is with Bazel 3.6.0 running inside a CentOS 7 container:

# /root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/BUILD:47:19
cc_toolchain_suite(
  name = "toolchain",
  tags = ["__CC_RULES_MIGRATION_DO_NOT_USE_WILL_BREAK__"],
  generator_name = "toolchain",
  generator_function = "cc_toolchain_suite",
  generator_location = "/root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/BUILD:47:19",
  toolchains = {"aarch64|compiler": "@local_config_cc//:cc-compiler-aarch64", "aarch64": "@local_config_cc//:cc-compiler-aarch64", "armeabi-v7a|compiler": "@local_config_cc//:cc-compiler-armeabi-v7a", "armeabi-v7a": "@local_config_cc//:cc-compiler-armeabi-v7a"},
)

# /root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/BUILD:25:11
cc_library(
  name = "malloc",
)

# /root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/BUILD:34:10
filegroup(
  name = "cc_wrapper",
  srcs = ["@local_config_cc//:cc_wrapper.sh"],
)

# /root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/BUILD:133:13
cc_toolchain(
  name = "cc-compiler-armeabi-v7a",
  tags = ["__CC_RULES_MIGRATION_DO_NOT_USE_WILL_BREAK__"],
  generator_name = "cc-compiler-armeabi-v7a",
  generator_function = "cc_toolchain",
  generator_location = "/root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/BUILD:133:13",
  all_files = "@local_config_cc//:empty",
  compiler_files = "@local_config_cc//:empty",
  strip_files = "@local_config_cc//:empty",
  objcopy_files = "@local_config_cc//:empty",
  as_files = "@local_config_cc//:empty",
  ar_files = "@local_config_cc//:empty",
  linker_files = "@local_config_cc//:empty",
  dwp_files = "@local_config_cc//:empty",
  supports_param_files = True,
  toolchain_identifier = "stub_armeabi-v7a",
  toolchain_config = "@local_config_cc//:stub_armeabi-v7a",
)

# /root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/BUILD:148:28
armeabi_cc_toolchain_config(
  name = "stub_armeabi-v7a",
)
# Rule armeabi_cc_toolchain_config defined at (most recent call last):
#   /root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/armeabi_cc_toolchain_config.bzl:78:35 in <toplevel>

# /root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/BUILD:57:13
cc_toolchain(
  name = "cc-compiler-aarch64",
  tags = ["__CC_RULES_MIGRATION_DO_NOT_USE_WILL_BREAK__"],
  generator_name = "cc-compiler-aarch64",
  generator_function = "cc_toolchain",
  generator_location = "/root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/BUILD:57:13",
  all_files = "@local_config_cc//:compiler_deps",
  compiler_files = "@local_config_cc//:compiler_deps",
  strip_files = "@local_config_cc//:empty",
  objcopy_files = "@local_config_cc//:empty",
  as_files = "@local_config_cc//:compiler_deps",
  ar_files = "@local_config_cc//:compiler_deps",
  linker_files = "@local_config_cc//:compiler_deps",
  dwp_files = "@local_config_cc//:empty",
  supports_param_files = True,
  toolchain_identifier = "local",
  toolchain_config = "@local_config_cc//:local",
)

# /root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/BUILD:73:20
cc_toolchain_config(
  name = "local",
  cpu = "aarch64",
  compiler = "compiler",
  toolchain_identifier = "local",
  host_system_name = "local",
  target_system_name = "local",
  target_libc = "local",
  abi_version = "local",
  abi_libc_version = "local",
  cxx_builtin_include_directories = ["/usr/lib/gcc/aarch64-redhat-linux/4.8.5/include", "/usr/local/include", "/usr/include", "/usr/include/c++/4.8.5", "/usr/include/c++/4.8.5/aarch64-redhat-linux", "/usr/include/c++/4.8.5/backward"],
  tool_paths = {"ar": "/usr/bin/ar", "ld": "/usr/bin/ld", "cpp": "/usr/bin/cpp", "gcc": "/usr/bin/gcc", "dwp": "/usr/bin/dwp", "gcov": "/usr/bin/gcov", "nm": "/usr/bin/nm", "objcopy": "/usr/bin/objcopy", "objdump": "/usr/bin/objdump", "strip": "/usr/bin/strip"},
  compile_flags = ["-U_FORTIFY_SOURCE", "-fstack-protector", "-Wall", "-Wunused-but-set-parameter", "-Wno-free-nonheap-object", "-fno-omit-frame-pointer"],
  dbg_compile_flags = ["-g"],
  opt_compile_flags = ["-g0", "-O2", "-D_FORTIFY_SOURCE=1", "-DNDEBUG", "-ffunction-sections", "-fdata-sections"],
  cxx_flags = ["-std=c++0x"],
  link_flags = ["-fuse-ld=gold", "-Wl,-no-as-needed", "-Wl,-z,relro,-z,now", "-B/usr/bin", "-pass-exit-codes", "-lstdc++", "-lm"],
  link_libs = [],
  opt_link_flags = ["-Wl,--gc-sections"],
  unfiltered_compile_flags = ["-fno-canonical-system-headers", "-Wno-builtin-macro-redefined", "-D__DATE__=\"redacted\"", "-D__TIMESTAMP__=\"redacted\"", "-D__TIME__=\"redacted\""],
  coverage_compile_flags = ["--coverage"],
  coverage_link_flags = ["--coverage"],
  supports_start_end_lib = True,
)
# Rule cc_toolchain_config defined at (most recent call last):
#   /root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/cc_toolchain_config.bzl:1231:27 in <toplevel>

# /root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/BUILD:29:10
filegroup(
  name = "empty",
  srcs = [],
)

# /root/.cache/bazel/_bazel_root/4539330648b80f94ef3bf911f6d77ac9/external/local_config_cc/BUILD:39:10
filegroup(
  name = "compiler_deps",
  srcs = ["@local_config_cc//:builtin_include_directory_paths"],
)

[fweikert]

Is this issue still relevant? I'm working on adding more ARM64 machines to our CI.

[philwo]

I think it would be good to have this - even if it might not strictly necessary when you have ARM64 machines for CI, it would serve as a good real world example of how to cross-compile a complex project such as Bazel itself from Linux x86_64 to Linux arm64. I know that I would certainly look at this in the future (instead of trying to figure it out via the documentation) if I wanted to setup something like this. And it would prove that it actually works well and reliably.