Skip to content

Commit 47cdf2c

Browse files
pkestenepkestene
authored
Merge pull request #9 from pkestene/refactor/mpi
Refactor mpi
2 parents 7e08927 + 02b184a commit 47cdf2c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+2675
-2794
lines changed

‎CMakeLists.txt

Lines changed: 37 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,18 @@
11
# set minimal version the one requested by kokkos
22
cmake_minimum_required(VERSION 3.18)
33

4+
# The ``target_sources()`` command converts relative paths to absolute.
5+
if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13.0")
6+
message(STATUS "Setting policy CMP0076 to use new behavior")
7+
cmake_policy(SET CMP0076 NEW)
8+
endif()
9+
10+
# CMake 3.24 and above prefers to set the timestamps of all extracted contents
11+
# to the time of the extraction.
12+
if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.24)
13+
cmake_policy(SET CMP0135 NEW)
14+
endif()
15+
416
#
517
# default local cmake macro repository
618
#
@@ -43,7 +55,22 @@ unset(PROJECT_LANGUAGES)
4355

4456
set(PROJECT_LANGUAGES ${PROJECT_LANGUAGES} C CXX)
4557

46-
project(euler_kokkos LANGUAGES ${PROJECT_LANGUAGES})
58+
set(EULER_KOKKOS_VERSION "0.9.0")
59+
60+
# deduce EULER_KOKKOS_SHORT_VERSION using regex
61+
string(REGEX MATCH "^[0-9]+\.[0-9]+\.[0-9]+" EULER_KOKKOS_SHORT_VERSION
62+
${EULER_KOKKOS_VERSION})
63+
if("${EULER_KOKKOS_SHORT_VERSION}" STREQUAL "")
64+
message(
65+
FATAL_ERROR
66+
"Unable to compute short version from EULER_KOKKOS_VERSION=${EULER_KOKKOS_VERSION}"
67+
)
68+
endif()
69+
70+
project(
71+
euler_kokkos
72+
VERSION ${EULER_KOKKOS_SHORT_VERSION}
73+
LANGUAGES ${PROJECT_LANGUAGES})
4774

4875
# Documentation type
4976
if(EULER_KOKKOS_BUILD_DOC)
@@ -67,55 +94,12 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
6794
set(CMAKE_CXX_STANDARD 17)
6895
set(CMAKE_CXX_EXTENSIONS OFF)
6996

97+
# ##############################################################################
7098
#
71-
# Write a small header with build data, git version, etc...
99+
# check MPI, VTK, HDF5, PNETCDF, ...
72100
#
73-
include(write_version)
74-
75-
# ##############################################################################
76-
# External packages: MPI, ...
77101
# ##############################################################################
78-
79-
# ##############################################################################
80-
# find_package(MPI REQUIRED)
81-
find_package(MPI)
82-
if(EULER_KOKKOS_USE_MPI)
83-
if(MPI_CXX_FOUND)
84-
message(STATUS "MPI support found")
85-
message(STATUS "MPI compile flags: " ${MPI_CXX_COMPILE_FLAGS})
86-
message(STATUS "MPI include path: " ${MPI_CXX_INCLUDE_PATH})
87-
message(STATUS "MPI LINK flags path: " ${MPI_CXX_LINK_FLAGS})
88-
message(STATUS "MPI libraries: " ${MPI_CXX_LIBRARIES})
89-
90-
# set(CMAKE_EXE_LINKER_FLAGS ${MPI_CXX_LINK_FLAGS})
91-
find_program(
92-
OMPI_INFO
93-
NAMES ompi_info
94-
HINTS ${MPI_CXX_LIBRARIES}/../bin)
95-
96-
# Full command line to probe if cuda support in MPI implementation is
97-
# enabled ompi_info --parsable --all | grep
98-
# mpi_built_with_cuda_support:value
99-
if(OMPI_INFO)
100-
execute_process(COMMAND ${OMPI_INFO} OUTPUT_VARIABLE _output)
101-
if((_output MATCHES "smcuda") OR (EULER_KOKKOS_USE_MPI_CUDA_AWARE_ENFORCED
102-
))
103-
set(MPI_CUDA_AWARE_ENABLED True)
104-
message(STATUS "Found OpenMPI with CUDA support built.")
105-
else()
106-
set(MPI_CUDA_AWARE_ENABLED False)
107-
message(WARNING "OpenMPI found, but it is not built with CUDA support.")
108-
add_compile_options(-DMPI_CUDA_AWARE_OFF)
109-
endif()
110-
endif()
111-
else()
112-
message(
113-
WARNING
114-
"Not compiling with MPI. Suppress this warning with -DEULER_KOKKOS_USE_MPI=OFF"
115-
)
116-
set(EULER_KOKKOS_USE_MPI OFF)
117-
endif()
118-
endif()
102+
include(cmake/config_mpi.cmake)
119103

120104
if(EULER_KOKKOS_USE_VTK)
121105
# look for VTK only if requested; VTK macro might even be not present on the
@@ -149,22 +133,7 @@ endif(EULER_KOKKOS_USE_VTK)
149133
# ##############################################################################
150134
# HDF5
151135
# ##############################################################################
152-
# prefer using parallel HDF5 when build with mpi
153-
if(EULER_KOKKOS_USE_MPI)
154-
set(HDF5_PREFER_PARALLEL TRUE)
155-
endif(EULER_KOKKOS_USE_MPI)
156-
157-
if(EULER_KOKKOS_USE_HDF5)
158-
find_package(HDF5)
159-
if(HDF5_FOUND)
160-
include_directories(${HDF5_INCLUDE_DIRS})
161-
set(MY_HDF5_LIBS hdf5 hdf5_cpp)
162-
add_compile_options(-DUSE_HDF5)
163-
if(HDF5_IS_PARALLEL)
164-
add_compile_options(-DUSE_HDF5_PARALLEL)
165-
endif()
166-
endif(HDF5_FOUND)
167-
endif(EULER_KOKKOS_USE_HDF5)
136+
include(cmake/config_hdf5.cmake)
168137

169138
# ##############################################################################
170139
# PNETCDF
@@ -184,38 +153,6 @@ endif(EULER_KOKKOS_USE_MPI)
184153
#
185154
include(build_or_find_kokkos)
186155

187-
#
188-
# if MPI is enabled and kokkos backend is cuda, we need a cuda aware
189-
# implementation
190-
#
191-
if(EULER_KOKKOS_USE_MPI)
192-
193-
if(MPI_FOUND
194-
AND (EULER_KOKKOS_BACKEND MATCHES "Cuda")
195-
AND (NOT MPI_CUDA_AWARE_ENABLED))
196-
197-
message(FATAL_ERROR "You must use a cuda-aware MPI implementation.")
198-
199-
endif()
200-
201-
endif()
202-
203-
#
204-
# common flags
205-
#
206-
207-
if(EULER_KOKKOS_USE_DOUBLE)
208-
add_compile_options(-DUSE_DOUBLE)
209-
endif()
210-
211-
if(EULER_KOKKOS_USE_MPI)
212-
add_compile_options(-DUSE_MPI)
213-
endif()
214-
215-
if(EULER_KOKKOS_USE_FPE_DEBUG)
216-
add_compile_options(-DUSE_FPE_DEBUG)
217-
endif()
218-
219156
#
220157
# Using flags -Wextra, it's to strong for Kokkos, too many warnings But -Wall is
221158
# really a minimum
@@ -225,6 +162,11 @@ endif()
225162
# -pedantic ) add_definitions( -Wall -Wextra )
226163
add_definitions(-Wall)
227164

165+
# Generate euler_kokkos_config.h and euler_kokkos_version.h (with git info and
166+
# build date)
167+
#
168+
include(cmake/generate_config_h.cmake)
169+
228170
#
229171
# sources
230172
#

‎cmake/CheckMPIFeatures.cmake

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
#
2+
# This file is borrowed and slightly modified from
3+
# https://github.com/eschnett/MPIwrapper/blob/main/cmake/CheckMPIFeatures.cmake
4+
#
5+
# function check_mpi_features provides helper to check if MPI implementation has
6+
# the runtime ability to probe GPU-awareness
7+
#
8+
# * cuda-aware (Nvidia GPU),
9+
# * hip-aware (AMD GPU),
10+
# * ze-aware (INTEL GPU)
11+
#
12+
# Apparently Intel MPI (as of version 2021.7.0) doesn't provide header
13+
# mpi-ext.h.
14+
#
15+
16+
include(CheckCSourceCompiles)
17+
18+
# check_mpi_features
19+
macro(CHECK_MPI_FEATURES)
20+
if(NOT DEFINED EULER_KOKKOS_USE_MPI_EXT
21+
OR NOT DEFINED EULER_KOKKOS_MPI_HAS_QUERY_CUDA_SUPPORT)
22+
list(JOIN MPI_COMPILE_FLAGS " " cmake_required_flags)
23+
24+
# set(cmake_required_includes ${MPI_INCLUDE_PATH})
25+
# set(cmake_required_libraries ${MPI_LIBRARIES})
26+
set(CMAKE_REQUIRED_LIBRARIES MPI::MPI_C)
27+
28+
# We cannot use check_include_file here as <mpi.h> needs to be included
29+
# before <mpi-ext.h>, and check_include_file doesn't support this.
30+
check_c_source_compiles(
31+
"
32+
#include <mpi.h>
33+
#include <mpi-ext.h>
34+
int main() {
35+
return 0;
36+
}
37+
"
38+
EULER_KOKKOS_USE_MPI_EXT FAIL_REGEX "")
39+
40+
if(NOT EULER_KOKKOS_USE_MPI_EXT)
41+
set(EULER_KOKKOS_USE_MPI_EXT 0)
42+
else()
43+
set(EULER_KOKKOS_USE_MPI_EXT 1)
44+
endif()
45+
46+
list(APPEND CMAKE_REQUIRED_DEFINITIONS
47+
-DEULER_KOKKOS_USE_MPI_EXT=${EULER_KOKKOS_USE_MPI_EXT})
48+
49+
check_c_source_compiles(
50+
"
51+
#include <mpi.h>
52+
#if EULER_KOKKOS_USE_MPI_EXT
53+
#include <mpi-ext.h>
54+
#endif
55+
int main() {
56+
int result = MPIX_Query_cuda_support();
57+
return 0;
58+
}
59+
"
60+
EULER_KOKKOS_MPI_HAS_QUERY_CUDA_SUPPORT)
61+
62+
if(NOT EULER_KOKKOS_MPI_HAS_QUERY_CUDA_SUPPORT)
63+
set(EULER_KOKKOS_MPI_HAS_QUERY_CUDA_SUPPORT 0)
64+
else()
65+
set(EULER_KOKKOS_MPI_HAS_QUERY_CUDA_SUPPORT 1)
66+
endif()
67+
68+
check_c_source_compiles(
69+
"
70+
#include <mpi.h>
71+
#if EULER_KOKKOS_USE_MPI_EXT
72+
#include <mpi-ext.h>
73+
#endif
74+
int main() {
75+
int result = MPIX_Query_hip_support();
76+
return 0;
77+
}
78+
"
79+
EULER_KOKKOS_MPI_HAS_QUERY_HIP_SUPPORT)
80+
81+
if(NOT EULER_KOKKOS_MPI_HAS_QUERY_HIP_SUPPORT)
82+
set(EULER_KOKKOS_MPI_HAS_QUERY_HIP_SUPPORT 0)
83+
else()
84+
set(EULER_KOKKOS_MPI_HAS_QUERY_HIP_SUPPORT 1)
85+
endif()
86+
87+
check_c_source_compiles(
88+
"
89+
#include <mpi.h>
90+
#if EULER_KOKKOS_USE_MPI_EXT
91+
#include <mpi-ext.h>
92+
#endif
93+
int main() {
94+
int result = MPIX_Query_ze_support();
95+
return 0;
96+
}
97+
"
98+
EULER_KOKKOS_MPI_HAS_QUERY_ZE_SUPPORT)
99+
100+
if(NOT EULER_KOKKOS_MPI_HAS_QUERY_ZE_SUPPORT)
101+
set(EULER_KOKKOS_MPI_HAS_QUERY_ZE_SUPPORT 0)
102+
else()
103+
set(EULER_KOKKOS_MPI_HAS_QUERY_ZE_SUPPORT 1)
104+
endif()
105+
106+
list(REMOVE_ITEM CMAKE_REQUIRED_DEFINITIONS -DEULER_KOKKOS_USE_MPI_EXT)
107+
endif()
108+
endmacro(CHECK_MPI_FEATURES)
109+
110+
check_mpi_features()

‎cmake/config_hdf5.cmake

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# ##############################################################################
2+
# HDF5
3+
# ##############################################################################
4+
# prefer using parallel HDF5 when build with mpi
5+
if(EULER_KOKKOS_USE_MPI)
6+
set(HDF5_PREFER_PARALLEL TRUE)
7+
endif(EULER_KOKKOS_USE_MPI)
8+
9+
if(EULER_KOKKOS_USE_HDF5)
10+
set(HDF5_FIND_DEBUG True)
11+
find_package(HDF5 COMPONENTS C CXX HL REQUIRED)
12+
if(HDF5_FOUND)
13+
set(EULER_KOKKOS_USE_HDF5_VERSION ${HDF5_VERSION})
14+
include_directories(${HDF5_INCLUDE_DIRS})
15+
set(MY_HDF5_LIBS hdf5 hdf5_cpp)
16+
if(HDF5_IS_PARALLEL)
17+
set(EULER_KOKKOS_USE_HDF5_PARALLEL True)
18+
message(WARNING "PARALLEL HDF5 found")
19+
else()
20+
message(WARNING "PARALLEL HDF5 not found")
21+
endif()
22+
else()
23+
set(EULER_KOKKOS_USE_HDF5_VERSION "")
24+
message(WARNING "HDF5 not found")
25+
endif(HDF5_FOUND)
26+
endif(EULER_KOKKOS_USE_HDF5)

‎cmake/config_mpi.cmake

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# ##############################################################################
2+
# MPI
3+
# ##############################################################################
4+
find_package(MPI)
5+
if(EULER_KOKKOS_USE_MPI)
6+
if(MPI_CXX_FOUND)
7+
message(STATUS "MPI support found")
8+
message(STATUS "MPI compile flags: " ${MPI_CXX_COMPILE_FLAGS})
9+
message(STATUS "MPI include path: " ${MPI_CXX_INCLUDE_PATH})
10+
message(STATUS "MPI LINK flags path: " ${MPI_CXX_LINK_FLAGS})
11+
message(STATUS "MPI libraries: " ${MPI_CXX_LIBRARIES})
12+
13+
# set(CMAKE_EXE_LINKER_FLAGS ${MPI_CXX_LINK_FLAGS})
14+
find_program(
15+
OMPI_INFO
16+
NAMES ompi_info
17+
HINTS ${MPI_CXX_LIBRARIES}/../bin)
18+
19+
# Full command line to probe if cuda support in MPI implementation is
20+
# enabled ompi_info --parsable --all | grep
21+
# mpi_built_with_cuda_support:value
22+
if(OMPI_INFO)
23+
execute_process(COMMAND ${OMPI_INFO} OUTPUT_VARIABLE _output)
24+
if((_output MATCHES "smcuda") OR (EULER_KOKKOS_USE_MPI_CUDA_AWARE_ENFORCED
25+
))
26+
set(EULER_KOKKOS_USE_MPI_CUDA_AWARE_ENABLED ON)
27+
message(STATUS "Found OpenMPI with CUDA support built.")
28+
else()
29+
set(EULER_KOKKOS_USE_MPI_CUDA_AWARE_ENABLED OFF)
30+
message(WARNING "OpenMPI found, but it is not built with CUDA support.")
31+
add_compile_options(-DMPI_CUDA_AWARE_OFF)
32+
endif()
33+
endif()
34+
35+
else()
36+
message(
37+
WARNING
38+
"Not compiling with MPI. Suppress this warning with -DEULER_KOKKOS_USE_MPI=OFF"
39+
)
40+
set(EULER_KOKKOS_USE_MPI OFF)
41+
endif()
42+
endif()
43+
44+
# test if mpi-ext.h is available
45+
include(cmake/CheckMPIFeatures.cmake)

0 commit comments

Comments
 (0)