CMake, HDF5: inconsistency generating file: H5Tinit.c

[phcerdan]

Compiling after re-configure fails:

cd build; cmake . ; cmake --build .;

 cmake --build .                                                                                    13:56:05 
[2/5] Building C object Modules/ThirdParty/HDF5/src/itkhdf5/src/CMakeFiles/hdf5-static.dir/__/H5Tinit.c.o
FAILED: Modules/ThirdParty/HDF5/src/itkhdf5/src/CMakeFiles/hdf5-static.dir/__/H5Tinit.c.o
/usr/lib/colorgcc/bin/cc -D_GNU_SOURCE -Ifftw/include -IModules/ThirdParty/ZLIB/src -IITK/src-ITK-master/Modules/ThirdParty/ZLIB/src -IITK/src-ITK-master/Modules/ThirdParty/HDF5/src -IModules/ThirdParty/HDF5/src -IITK/src-ITK-master/Modules/ThirdParty/HDF5/src/itkhdf5/src -IModules/ThirdParty/HDF5/src/itkhdf5 -std=c99  -mtune=native -march=corei7 -Wno-uninitialized -Wno-unused-parameter -Wall -Wcast-align -Wdisabled-optimization -Wextra -Wformat=2 -Winvalid-pch -Wno-format-nonliteral -Wpointer-arith -Wshadow -Wunused -Wwrite-strings -funit-at-a-time -Wno-strict-overflow  -w -fstdarg-opt -pedantic -Wall -Wextra -fmessage-length=0 -O3 -DNDEBUG -fPIC -std=gnu99 -MD -MT Modules/ThirdParty/HDF5/src/itkhdf5/src/CMakeFiles/hdf5-static.dir/__/H5Tinit.c.o -MF Modules/ThirdParty/HDF5/src/itkhdf5/src/CMakeFiles/hdf5-static.dir/__/H5Tinit.c.o.d -o Modules/ThirdParty/HDF5/src/itkhdf5/src/CMakeFiles/hdf5-static.dir/__/H5Tinit.c.o -c Modules/ThirdParty/HDF5/src/itkhdf5/H5Tinit.c
cc: error: Modules/ThirdParty/HDF5/src/itkhdf5/H5Tinit.c: No such file or directory
cc: fatal error: no input files
compilation terminated.
[3/5] Linking CXX static library lib/libITKCommon-5.2.a
ninja: build stopped: subcommand failed.

But if you reconfigure again (twice), it works, because triggers regeneration of the HDF file:

[3/8] Generating ../H5Tinit.c, ../gen_SRCS.stamp1
[8/8] All good

Changes on HDF5 where introduced here: https://github.com/InsightSoftwareConsortium/ITK/commit/bbc7945343e9d9f697008d822dc866c20687bae8

if (HDF5_BATCH_H5DETECT)
  configure_file (
      ${HDF5_SOURCE_DIR}/bin/batch/${HDF5_BATCH_H5DETECT_SCRIPT}.in.cmake
      ${HDF5_BINARY_DIR}/${HDF5_BATCH_H5DETECT_SCRIPT} ESCAPE_QUOTES @ONLY
  )
  add_custom_command (
      OUTPUT ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c
             ${HDF5_GENERATED_SOURCE_DIR}/gen_SRCS.stamp1
      COMMAND ${HDF5_BATCH_CMD}
      ARGS ${HDF5_BINARY_DIR}/${HDF5_BATCH_H5DETECT_SCRIPT}
      COMMAND ${CMAKE_COMMAND}
      ARGS -E echo "Executed batch command to create H5Tinit.c"
      COMMAND    ${CMAKE_COMMAND}
      ARGS       -E touch ${HDF5_GENERATED_SOURCE_DIR}/gen_SRCS.stamp1
      DEPENDS H5detect
      WORKING_DIRECTORY ${HDF5_GENERATED_SOURCE_DIR}
  )
  add_custom_target (gen_H5Tinit
      COMMAND ${CMAKE_COMMAND} -P ${HDF5_SOURCE_DIR}/config/cmake/wait_H5Tinit.cmake
  )
  if (BUILD_SHARED_LIBS)
    add_custom_command (
        OUTPUT     ${HDF5_GENERATED_SOURCE_DIR}/shared/H5Tinit.c
                   ${HDF5_GENERATED_SOURCE_DIR}/shared/shared_gen_SRCS.stamp1
        COMMAND    ${CMAKE_COMMAND}
        ARGS       -E copy_if_different "${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c" "${HDF5_GENERATED_SOURCE_DIR}/shared/H5Tinit.c"
        COMMAND    ${CMAKE_COMMAND}
        ARGS       -E touch ${HDF5_GENERATED_SOURCE_DIR}/shared/shared_gen_SRCS.stamp1
        DEPENDS    gen_H5Tinit ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit_created
        WORKING_DIRECTORY ${HDF5_GENERATED_SOURCE_DIR}
    )
    set_source_files_properties (${HDF5_GENERATED_SOURCE_DIR}/shared/H5Tinit.c PROPERTIES GENERATED TRUE)
  endif ()
else ()
  add_custom_command (
      OUTPUT ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c
             ${HDF5_GENERATED_SOURCE_DIR}/gen_SRCS.stamp1
      COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:H5detect>
      ARGS ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c
      COMMAND    ${CMAKE_COMMAND}
      ARGS       -E touch ${HDF5_GENERATED_SOURCE_DIR}/gen_SRCS.stamp1
      DEPENDS H5detect
      WORKING_DIRECTORY ${HDF5_GENERATED_SOURCE_DIR}
  )

[hjmjohnson]

@phcerdan This one has been hard to find as the behavior changes for static vs shared vs changing between vs multi-config vs standard config.

Can you send a more complete picture of your build environment? Linux, CMake version, -G Ninja, USE_SHARED=ON|OFF ?

Thanks, Hans

[phcerdan]

Thanks @hjmjohnson , hope this helps:

Linux (arch)
cmake version 3.19.1
BUILD_SHARED_LIBS=OFF
-G Ninja
...
In case it helps too: No examples, no tests, and CMAKE_BUILD_TYPE: RelWithDebInfo

cmake -G Ninja -DBUILD_SHARED_LIBS=OFF ../itk-src

[phcerdan]

Just wondering:

  add_custom_command (
      OUTPUT ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c
             ${HDF5_GENERATED_SOURCE_DIR}/gen_SRCS.stamp1
      COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:H5detect>
      ARGS ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c
      COMMAND    ${CMAKE_COMMAND}
      ARGS       -E touch ${HDF5_GENERATED_SOURCE_DIR}/gen_SRCS.stamp1
      DEPENDS H5detect ### <------ change for gen_H5Tinit
      WORKING_DIRECTORY ${HDF5_GENERATED_SOURCE_DIR}
  )

testing it.

[hjmjohnson]

@phcerdan My other thought is that perhaps the 2 COMMANDS are not run in series such that perhaps the 'touch' occurs before the 'H5detect'.

[phcerdan]

I have solved it removing the whole else that runs when ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c exists. So, CMake always recreates the file (and the stamps).

So all this out: https://github.com/InsightSoftwareConsortium/ITK/blob/92b891d33ae9ee4fcd95055e74dff24a6b114d36/Modules/ThirdParty/HDF5/src/itkhdf5/src/CMakeLists.txt#L1017-L1037

What do you think? ninja doesn't do extra work, but I haven't tested with make

[hjmjohnson]

Is this going to cause rebuilds all the time.

[phcerdan]

Is this going to cause rebuilds all the time.

Was that a question @hjmjohnson?

In my case (ninja, and ccache) it doesn't.

[stale[bot]]

This issue has been automatically marked as stale because it has not had recent activity. Thank you for your contributions.

[hjmjohnson]

This problem has resurfaced again. On mac build, every other cmake config causes the file to be removed.

There is an indication that this is fixed in upstream versions of HDF5 where the "byproduct" option is used. https://gitlab.kitware.com/paraview/paraview/-/issues/19787

Note changes need to be tested for @thewtex changes with emscripten.

[seanm]

I have this to bring ITK closer to HDF5 upstream: https://github.com/InsightSoftwareConsortium/ITK/pull/2832 but I'm rather stuck on it...

[stale[bot]]

This issue has been automatically marked as stale because it has not had recent activity. Thank you for your contributions.

[dzenanz]

Is this still an issue @phcerdan?