Files
deeplearning4j--deeplearning4j/libnd4j/cmake/PostBuild.cmake
T
2026-07-13 12:47:05 +08:00

400 lines
17 KiB
CMake

# cmake/PostBuild.cmake
# Configures optional post-build targets like tests and analysis tools.
# ============================================================================
# HELPER FUNCTION: Write file only if content changed (preserves mtime for PCH)
# This prevents PCH invalidation during builds when config.h content is identical
# ============================================================================
function(_postbuild_write_if_different filepath content)
set(should_write TRUE)
# Check if file already exists
if(EXISTS "${filepath}")
# Read existing content
file(READ "${filepath}" existing_content)
# Compare content
if("${existing_content}" STREQUAL "${content}")
set(should_write FALSE)
endif()
endif()
# Only write if content changed or file doesn't exist
if(should_write)
file(WRITE "${filepath}" "${content}")
endif()
endfunction()
# --- Configuration file ---
# Build TYPE_DEFINES string from current type configuration
# This ensures JavaCPP sees the same type availability as the CMake build
set(TYPE_DEFINES "")
# Check if we're in selective types mode
if(DEFINED SD_TYPES_LIST AND NOT SD_TYPES_LIST STREQUAL "")
# Selective types mode - export only enabled types
# The HAS_* macros were already set by apply_type_definitions_to_target
# We need to capture them and add to TYPE_DEFINES
# Get all HAS_* definitions that were set
get_directory_property(COMPILE_DEFS COMPILE_DEFINITIONS)
foreach(def IN LISTS COMPILE_DEFS)
if(def MATCHES "^HAS_")
string(APPEND TYPE_DEFINES "#define ${def}\n")
endif()
endforeach()
else()
# All types mode - define all HAS_* macros
string(APPEND TYPE_DEFINES "#define HAS_BOOL 1\n")
string(APPEND TYPE_DEFINES "#define HAS_FLOAT 1\n")
string(APPEND TYPE_DEFINES "#define HAS_FLOAT32 1\n")
string(APPEND TYPE_DEFINES "#define HAS_DOUBLE 1\n")
string(APPEND TYPE_DEFINES "#define HAS_FLOAT64 1\n")
string(APPEND TYPE_DEFINES "#define HAS_FLOAT16 1\n")
string(APPEND TYPE_DEFINES "#define HAS_HALF 1\n")
string(APPEND TYPE_DEFINES "#define HAS_BFLOAT16 1\n")
string(APPEND TYPE_DEFINES "#define HAS_BFLOAT 1\n")
string(APPEND TYPE_DEFINES "#define HAS_FLOAT8 1\n")
string(APPEND TYPE_DEFINES "#define HAS_HALF2 1\n")
string(APPEND TYPE_DEFINES "#define HAS_INT8 1\n")
string(APPEND TYPE_DEFINES "#define HAS_INT8_T 1\n")
string(APPEND TYPE_DEFINES "#define HAS_INT16 1\n")
string(APPEND TYPE_DEFINES "#define HAS_INT16_T 1\n")
string(APPEND TYPE_DEFINES "#define HAS_INT32 1\n")
string(APPEND TYPE_DEFINES "#define HAS_INT32_T 1\n")
string(APPEND TYPE_DEFINES "#define HAS_INT 1\n")
string(APPEND TYPE_DEFINES "#define HAS_INT64 1\n")
string(APPEND TYPE_DEFINES "#define HAS_INT64_T 1\n")
string(APPEND TYPE_DEFINES "#define HAS_LONG 1\n")
string(APPEND TYPE_DEFINES "#define HAS_UINT8 1\n")
string(APPEND TYPE_DEFINES "#define HAS_UINT8_T 1\n")
string(APPEND TYPE_DEFINES "#define HAS_UINT16 1\n")
string(APPEND TYPE_DEFINES "#define HAS_UINT16_T 1\n")
string(APPEND TYPE_DEFINES "#define HAS_UINT32 1\n")
string(APPEND TYPE_DEFINES "#define HAS_UINT32_T 1\n")
string(APPEND TYPE_DEFINES "#define HAS_UINT64 1\n")
string(APPEND TYPE_DEFINES "#define HAS_UINT64_T 1\n")
string(APPEND TYPE_DEFINES "#define HAS_UNSIGNEDLONG 1\n")
string(APPEND TYPE_DEFINES "#define HAS_UTF8 1\n")
string(APPEND TYPE_DEFINES "#define HAS_STD_STRING 1\n")
string(APPEND TYPE_DEFINES "#define HAS_UTF16 1\n")
string(APPEND TYPE_DEFINES "#define HAS_STD_U16STRING 1\n")
string(APPEND TYPE_DEFINES "#define HAS_UTF32 1\n")
string(APPEND TYPE_DEFINES "#define HAS_STD_U32STRING 1\n")
endif()
# Generate config.h with write-if-different to preserve PCH
# This prevents PCH invalidation during the build when content hasn't changed
set(CONFIG_H_IN "${CMAKE_CURRENT_SOURCE_DIR}/include/config.h.in")
set(CONFIG_H_OUT "${CMAKE_CURRENT_BINARY_DIR}/include/config.h")
# Read the template
file(READ "${CONFIG_H_IN}" CONFIG_H_CONTENT)
# Perform variable substitutions (mimics configure_file behavior)
# Handle cmakedefine01 directives
if(HAVE_ONEDNN)
string(REPLACE "#cmakedefine01 HAVE_ONEDNN" "#define HAVE_ONEDNN 1" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
else()
string(REPLACE "#cmakedefine01 HAVE_ONEDNN" "#define HAVE_ONEDNN 0" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
endif()
if(HAVE_ARMCOMPUTE)
string(REPLACE "#cmakedefine01 HAVE_ARMCOMPUTE" "#define HAVE_ARMCOMPUTE 1" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
else()
string(REPLACE "#cmakedefine01 HAVE_ARMCOMPUTE" "#define HAVE_ARMCOMPUTE 0" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
endif()
if(HAVE_CUDNN)
string(REPLACE "#cmakedefine01 HAVE_CUDNN" "#define HAVE_CUDNN 1" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
else()
string(REPLACE "#cmakedefine01 HAVE_CUDNN" "#define HAVE_CUDNN 0" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
endif()
if(HAVE_OPENBLAS)
string(REPLACE "#cmakedefine01 HAVE_OPENBLAS" "#define HAVE_OPENBLAS 1" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
else()
string(REPLACE "#cmakedefine01 HAVE_OPENBLAS" "#define HAVE_OPENBLAS 0" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
endif()
if(HAVE_FLATBUFFERS)
string(REPLACE "#cmakedefine01 HAVE_FLATBUFFERS" "#define HAVE_FLATBUFFERS 1" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
else()
string(REPLACE "#cmakedefine01 HAVE_FLATBUFFERS" "#define HAVE_FLATBUFFERS 0" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
endif()
if(SD_SELECTIVE_TYPES)
string(REPLACE "#cmakedefine01 SD_SELECTIVE_TYPES" "#define SD_SELECTIVE_TYPES 1" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
else()
string(REPLACE "#cmakedefine01 SD_SELECTIVE_TYPES" "#define SD_SELECTIVE_TYPES 0" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
endif()
# Handle @VAR@ substitutions
if(DEFINED SD_LIBRARY_NAME)
string(REPLACE "@SD_LIBRARY_NAME@" "${SD_LIBRARY_NAME}" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
else()
# Remove the line if variable not defined
string(REGEX REPLACE "#cmakedefine SD_LIBRARY_NAME[^\n]*\n" "" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
endif()
if(DEFINED ONEDNN_PATH)
string(REPLACE "@ONEDNN_PATH@" "${ONEDNN_PATH}" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
else()
string(REGEX REPLACE "#cmakedefine ONEDNN_PATH[^\n]*\n" "" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
endif()
if(DEFINED OPENBLAS_PATH)
string(REPLACE "@OPENBLAS_PATH@" "${OPENBLAS_PATH}" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
else()
string(REGEX REPLACE "#cmakedefine OPENBLAS_PATH[^\n]*\n" "" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
endif()
if(DEFINED FLATBUFFERS_PATH)
string(REPLACE "@FLATBUFFERS_PATH@" "${FLATBUFFERS_PATH}" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
else()
string(REGEX REPLACE "#cmakedefine FLATBUFFERS_PATH[^\n]*\n" "" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
endif()
if(DEFINED DEFAULT_ENGINE)
string(REPLACE "@DEFAULT_ENGINE@" "${DEFAULT_ENGINE}" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
else()
string(REGEX REPLACE "#cmakedefine DEFAULT_ENGINE[^\n]*\n" "" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
endif()
# Substitute TYPE_DEFINES
string(REPLACE "@TYPE_DEFINES@" "${TYPE_DEFINES}" CONFIG_H_CONTENT "${CONFIG_H_CONTENT}")
# Write config.h only if content changed (preserves mtime for PCH)
_postbuild_write_if_different("${CONFIG_H_OUT}" "${CONFIG_H_CONTENT}")
include_directories(${CMAKE_CURRENT_BINARY_DIR}/include)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
# --- Flatbuffers Header and Java Generation ---
if(DEFINED ENV{GENERATE_FLATC} OR DEFINED GENERATE_FLATC)
set(FLATC_EXECUTABLE "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-build/flatc")
set(MAIN_GENERATED_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/include/graph/generated.h")
add_custom_command(
OUTPUT ${MAIN_GENERATED_HEADER}
COMMAND ${CMAKE_COMMAND} -E env "FLATC_PATH=${FLATC_EXECUTABLE}" bash ${CMAKE_CURRENT_SOURCE_DIR}/flatc-generate.sh
DEPENDS flatbuffers_external
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "Running flatc to generate C++ headers"
VERBATIM
)
add_custom_target(generate_flatbuffers_headers DEPENDS ${MAIN_GENERATED_HEADER})
add_custom_command(
TARGET generate_flatbuffers_headers POST_BUILD
COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/copy-flatc-java.sh
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "Copying generated Java files"
VERBATIM
)
endif()
if(SD_EXTRACT_INSTANTIATIONS)
include(ExtractInstantiations)
endif()
# --- Self-Contained Preprocessing Target ---
if(SD_PREPROCESS)
message("Preprocessing enabled: ${CMAKE_BINARY_DIR}")
message("Setting up self-contained preprocessing with FlatBuffers...")
include(ExternalProject)
# Build FlatBuffers for preprocessing (minimal build, just need headers)
set(PREPROCESS_FLATBUFFERS_VERSION "25.2.10")
set(PREPROCESS_FLATBUFFERS_URL "https://github.com/google/flatbuffers/archive/v${PREPROCESS_FLATBUFFERS_VERSION}.tar.gz")
set(PREPROCESS_FB_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/preprocess-flatbuffers-src")
set(PREPROCESS_FB_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/preprocess-flatbuffers-build")
# Check if we already have FlatBuffers headers
set(PREPROCESS_FB_HEADER "${PREPROCESS_FB_SOURCE_DIR}/include/flatbuffers/flatbuffers.h")
if(NOT EXISTS ${PREPROCESS_FB_HEADER})
message("Downloading and extracting FlatBuffers for preprocessing...")
# Download and extract FlatBuffers
file(DOWNLOAD ${PREPROCESS_FLATBUFFERS_URL}
"${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-${PREPROCESS_FLATBUFFERS_VERSION}.tar.gz"
SHOW_PROGRESS
STATUS download_status)
list(GET download_status 0 download_result)
if(NOT download_result EQUAL 0)
message(FATAL_ERROR "Failed to download FlatBuffers for preprocessing")
endif()
# Extract the archive
execute_process(
COMMAND ${CMAKE_COMMAND} -E tar xzf "flatbuffers-${PREPROCESS_FLATBUFFERS_VERSION}.tar.gz"
WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
RESULT_VARIABLE extract_result
)
if(NOT extract_result EQUAL 0)
message(FATAL_ERROR "Failed to extract FlatBuffers for preprocessing")
endif()
# Move the extracted directory to our expected location
file(RENAME "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers-${PREPROCESS_FLATBUFFERS_VERSION}"
"${PREPROCESS_FB_SOURCE_DIR}")
endif()
# Verify FlatBuffers headers are now available
if(NOT EXISTS ${PREPROCESS_FB_HEADER})
message(FATAL_ERROR "FlatBuffers headers not found for preprocessing: ${PREPROCESS_FB_HEADER}")
endif()
message("✅ FlatBuffers headers available for preprocessing: ${PREPROCESS_FB_SOURCE_DIR}/include")
# Get ALL subdirectories under include/ and add the root include directory first
set(all_includes "${CMAKE_CURRENT_SOURCE_DIR}/include")
file(GLOB_RECURSE all_dirs LIST_DIRECTORIES true "${CMAKE_CURRENT_SOURCE_DIR}/include/*")
foreach(item ${all_dirs})
if(IS_DIRECTORY ${item})
list(APPEND all_includes ${item})
endif()
endforeach()
# Add binary include directory
list(APPEND all_includes "${CMAKE_CURRENT_BINARY_DIR}/include")
# Also add build directories
list(APPEND all_includes
"${CMAKE_SOURCE_DIR}/include/"
"${CMAKE_SOURCE_DIR}/include/system"
"${CMAKE_BINARY_DIR}/compilation_units"
"${CMAKE_BINARY_DIR}/cpu_instantiations"
"${CMAKE_BINARY_DIR}/cuda_instantiations"
"${CMAKE_BINARY_DIR}/include"
)
# Add our self-contained FlatBuffers include directory
list(APPEND all_includes "${PREPROCESS_FB_SOURCE_DIR}/include")
# Build include flags
set(include_flags "")
foreach(dir IN LISTS all_includes)
if(EXISTS ${dir})
string(APPEND include_flags " -I${dir}")
endif()
endforeach()
set(defs_flags "")
# Add basic platform definitions
string(APPEND defs_flags " -D__CPUBLAS__=true")
if(SD_CUDA)
string(APPEND defs_flags " -D__CUDABLAS__=true -DHAVE_CUDA=1")
endif()
if(HAVE_OPENBLAS)
string(APPEND defs_flags " -DHAVE_OPENBLAS=1")
endif()
if(SD_ALL_OPS OR "${SD_OPS_LIST}" STREQUAL "")
string(APPEND defs_flags " -DSD_ALL_OPS=1")
# When SD_ALL_OPS=1, NOT_EXCLUDED should evaluate to 1 for all ops
string(APPEND defs_flags " -DNOT_EXCLUDED(x)=1")
else()
# Add specific operation definitions
foreach(OP ${SD_OPS_LIST})
string(APPEND defs_flags " -DOP_${OP}=1")
endforeach()
# Define NOT_EXCLUDED macro for selective ops
string(APPEND defs_flags " -DNOT_EXCLUDED(x)=\\(defined\\(x\\)\\)")
endif()
# Add build type definitions
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
string(APPEND defs_flags " -DDEBUG=1 -D_DEBUG=1")
else()
string(APPEND defs_flags " -DNDEBUG=1")
endif()
# Add any additional compile definitions from the main build
if(compile_defs AND NOT compile_defs STREQUAL "compile_defs-NOTFOUND")
foreach(def IN LISTS compile_defs)
string(APPEND defs_flags " -D${def}")
endforeach()
endif()
set(PREPROCESSED_DIR "${CMAKE_SOURCE_DIR}/preprocessed")
file(MAKE_DIRECTORY ${PREPROCESSED_DIR})
set(PREPROCESSED_FILES)
set(PROCESSED_SOURCES "")
# Use ALL_SOURCES from global scope
if(DEFINED ALL_SOURCES AND ALL_SOURCES)
set(SOURCE_LIST ${ALL_SOURCES})
elseif(DEFINED ALL_SOURCES_LIST AND ALL_SOURCES_LIST)
set(SOURCE_LIST ${ALL_SOURCES_LIST})
else()
message(WARNING "No source list available for preprocessing. Skipping.")
add_custom_target(preprocess_sources ALL
COMMAND ${CMAKE_COMMAND} -E echo "Preprocessing skipped - no source files available")
return()
endif()
list(LENGTH SOURCE_LIST source_count)
message("Starting preprocessing of ${source_count} source files...")
message("Operation definitions: SD_ALL_OPS=${SD_ALL_OPS}, SD_OPS_LIST=${SD_OPS_LIST}")
foreach(src IN LISTS SOURCE_LIST)
if(NOT EXISTS ${src} OR NOT src MATCHES "\\.(c|cpp|cxx|cc|cu)$")
continue()
endif()
if(NOT src IN_LIST PROCESSED_SOURCES)
get_filename_component(src_name ${src} NAME_WE)
get_filename_component(src_path ${src} PATH)
file(RELATIVE_PATH rel_path ${CMAKE_SOURCE_DIR} ${src_path})
string(REPLACE "/" "_" src_dir_ "${rel_path}")
set(preprocessed_file "${PREPROCESSED_DIR}/${src_dir_}_${src_name}.i")
if(NOT EXISTS "${preprocessed_file}")
if(src MATCHES "\\.cu$")
set(compiler "${CMAKE_CUDA_COMPILER}")
set(lang_flags "${CMAKE_CUDA_FLAGS} ${CMAKE_CUDA_FLAGS_${CMAKE_BUILD_TYPE}}")
elseif(src MATCHES "\\.c$")
set(compiler "${CMAKE_C_COMPILER}")
set(lang_flags "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}}")
else()
set(compiler "${CMAKE_CXX_COMPILER}")
set(lang_flags "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE}}")
endif()
# Split the flags properly for execute_process
separate_arguments(lang_flags_list UNIX_COMMAND "${lang_flags}")
separate_arguments(defs_flags_list UNIX_COMMAND "${defs_flags}")
separate_arguments(include_flags_list UNIX_COMMAND "${include_flags}")
execute_process(
COMMAND ${compiler} -E -P -C ${lang_flags_list} ${defs_flags_list} ${include_flags_list} "${src}" -o "${preprocessed_file}"
RESULT_VARIABLE result
ERROR_VARIABLE error_output
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
)
if(result EQUAL 0)
list(APPEND PREPROCESSED_FILES ${preprocessed_file})
message(STATUS "✅ Preprocessed: ${src}")
else()
message(WARNING "❌ Failed to preprocess ${src}")
message(WARNING " Error: ${error_output}")
endif()
else()
list(APPEND PREPROCESSED_FILES ${preprocessed_file})
message(STATUS "✓ Already preprocessed: ${src}")
endif()
list(APPEND PROCESSED_SOURCES ${src})
endif()
endforeach()
list(LENGTH PREPROCESSED_FILES processed_count)
message("✅ Preprocessing complete. Generated ${processed_count} preprocessed files in ${PREPROCESSED_DIR}")
add_custom_target(preprocess_sources ALL DEPENDS ${PREPROCESSED_FILES})
endif()