Improvement of delayed allocation mechanism

.github/workflows/build.yml

@@ -62,9 +62,9 @@ jobs:
           - name: linux nvhpc-24.5
             os: ubuntu-22.04
             compiler: nvhpc-24.5
-            compiler_cc: mpicc
-            compiler_cxx: mpic++
-            compiler_fc: mpifort
+            compiler_cc: nvc
+            compiler_cxx: nvc++
+            compiler_fc: nvfortran
             python-version: '3.8'
             caching: true
 

CMakeLists.txt

@@ -82,6 +82,12 @@ ecbuild_add_option( FEATURE FIELD_GANG
     DEFAULT ON
 )
 
+##Get_view abort
+ecbuild_add_option( FEATURE GET_VIEW_ABORT
+	DESCRIPTION "Enable this option to make get_view abort when data are not on CPU"
+	DEFAULT ON
+)
+
 
 ## fypp preprocessor flags
 if(HAVE_BUDDY_MALLOC)

Readme.md

@@ -50,6 +50,7 @@ Features of FIELD_API can be toggled by passing the following argument to the CM
 | DOUBLE_PRECISION | ON | Enable the compilation of field_api in double precision |
 | CUDA | OFF | Enable the use of CUDA for GPU offload. Disables the use of the buddy memory allocator, removes the shadow host allocation for `FIELD%DEVPTR` and allocates owned fields (see below) in pinned (page-locked) host memory.|
 | FIELD_GANG | ON | Enable packed storage of groups of fields. This feature is not supported for the Cray compiler as it cannot resolve the underlying polymorphism.|
+| GET_VIEW_ABORT | ON | If activated, get_view will abort when the data are not present on CPU. |
 
 ## Supported compilers
 The library has been tested with the nvhpc toolkit from Nvidia, version 23.9/24.5
@@ -155,7 +156,8 @@ would then happen only if the data would be requested at some point, later in
 the program. It can be useful if one doesn't want to waste memory on data that
 might be only conditionally used. But please keep in mind, that allocating data
 can be slow and will slow down the program if done during a computation heavy
-part of the code.
+part of the code. The default value for the delayed option is false, but it can
+be switched by setting delayed\_default\_value to true.
 
 ```
 SUBROUTINE SUB(MYTEST)
@@ -280,13 +282,6 @@ write(*,*)"Total/Avg Time spend on transfer CPU->GPU", NUM_CPU_GPU_TR, "/" AVG,
 ...
 ```
 
-## Note on GET\_VIEW
-
-GET\_VIEW must only be called in sections of code running on the host. The
-field's data must be present on the host. It will not work if the data are on
-the device or if the field has not been allocated yet (when using the DELAY
-option).
-
 ## Cloning fields with FIELD\_CLONE\_ON_
 
 The subroutines FIELD_CLONE_ON_HOST and FIELD_CLONE_ON_DEVICE let a field be

python_utils/offload_backends/nvhpc/openacc.py

@@ -265,3 +265,13 @@ def end_data_deviceptr(cls):
 
 
         return "!$acc end data"
+
+    @classmethod
+    def reinit_gpu_context(cls):
+        """
+        Used to force reinitialization of GPU device.
+        Usefull when not calling GPU transfer function from OpenMP master thread
+        To use it you must have called the method *runtime_api_import* before
+        """
+
+        return "CALL ACC_SET_DEVICE_NUM(ACC_GET_DEVICE_NUM(ACC_DEVICE_NVIDIA), ACC_DEVICE_NVIDIA)"

python_utils/offload_backends/nvhpc/openmp.py

@@ -271,3 +271,13 @@ def update_host(cls, data):
         """
 
         return f"!$omp target update from ({','.join(data)})"
+
+    @classmethod
+    def reinit_gpu_context(cls):
+        """
+        Used to force reinitialization of GPU device.
+        Usefull when not calling GPU transfer function from OpenMP master thread
+        To use it you must have called the method *runtime_api_import* before
+        """
+
+        return "CALL OMP_SET_DEFAULT_DEVICE(OMP_GET_DEFAULT_DEVICE())"

python_utils/offload_macros.py

@@ -582,3 +582,15 @@ def memcpy_2D_intf(indent=0):
     method = _get_method(backend, 'memcpy_2D_intf')
 
     return _format_lines(method(), indent=indent)
+
+def reinit_gpu_context(indent=0):
+    """
+    Used to force reinitialization of GPU device.
+    Usefull when not calling GPU transfer function from OpenMP master thread
+    To use it you must have called the method *runtime_api_import* before
+    """
+
+    backend = _get_offload_backend()
+    method = _get_method(backend, 'reinit_gpu_context')
+
+    return _format_lines(method(), indent=indent)

src/buffer/field_RANKSUFF_gang_module.fypp

@@ -56,6 +56,8 @@ CONTAINS
   PROCEDURE :: CREATE_DEVICE_DATA => ${ftn}$_GANG_${type}$_CREATE_DEVICE_DATA
   PROCEDURE :: DELETE_DEVICE_DATA => ${ftn}$_GANG_${type}$_DELETE_DEVICE_DATA
   PROCEDURE :: SET_STATUS => ${ftn}$_GANG_${type}$_SET_STATUS
+  PROCEDURE :: ADD_STATUS => ${ftn}$_GANG_${type}$_ADD_STATUS
+  PROCEDURE :: REMOVE_STATUS => ${ftn}$_GANG_${type}$_REMOVE_STATUS
 END TYPE ${ftn}$_GANG_${type}$
 
 PUBLIC :: ${ftn}$_GANG_${type}$
@@ -277,6 +279,34 @@ CONTAINS
 
   END SUBROUTINE
 
+  SUBROUTINE ${ftn}$_GANG_${type}$_ADD_STATUS (SELF, KSTATUS)
+    CLASS(${ftn}$_GANG_${type}$) :: SELF
+    INTEGER (KIND=JPIM), INTENT (IN) :: KSTATUS
+
+    INTEGER (KIND=JPIM) :: JFLD
+
+    CALL SELF%${ftn}$_${type}$%ADD_STATUS (KSTATUS)
+
+    DO JFLD = 1, SIZE (SELF%CHILDREN)
+      CALL SELF%CHILDREN(JFLD)%PTR%ADD_STATUS (KSTATUS)
+    ENDDO
+
+  END SUBROUTINE
+
+  SUBROUTINE ${ftn}$_GANG_${type}$_REMOVE_STATUS (SELF, KSTATUS)
+    CLASS(${ftn}$_GANG_${type}$) :: SELF
+    INTEGER (KIND=JPIM), INTENT (IN) :: KSTATUS
+
+    INTEGER (KIND=JPIM) :: JFLD
+
+    CALL SELF%${ftn}$_${type}$%REMOVE_STATUS (KSTATUS)
+
+    DO JFLD = 1, SIZE (SELF%CHILDREN)
+      CALL SELF%CHILDREN(JFLD)%PTR%REMOVE_STATUS (KSTATUS)
+    ENDDO
+
+  END SUBROUTINE
+
 #:endfor
 
 #:endfor

src/core/CMakeLists.txt

@@ -52,6 +52,13 @@ field_api_add_object_library(
     LIBRARIES      field_api_debug
 )
 
+if(HAVE_GET_VIEW_ABORT)
+  target_compile_definitions( field_api_core PRIVATE "-DGET_VIEW_ABORT_DEFAULT_VALUE=.TRUE.")
+else()
+  target_compile_definitions( field_api_core PRIVATE "-DGET_VIEW_ABORT_DEFAULT_VALUE=.FALSE.")
+endif()
+
+
 set_source_files_properties( ${CMAKE_CURRENT_BINARY_DIR}/dev_alloc_module.F90 PROPERTIES COMPILE_OPTIONS $<${HAVE_CUDA}:-cuda>)
 set_source_files_properties( ${CMAKE_CURRENT_BINARY_DIR}/host_alloc_module.F90 PROPERTIES COMPILE_OPTIONS $<${HAVE_CUDA}:-cuda>)
 

src/core/dev_alloc_module.fypp

@@ -17,12 +17,6 @@ USE, INTRINSIC :: ISO_C_BINDING
 
 IMPLICIT NONE
 
-INTERFACE DEV_ALLOCATE_HST
-#:for ft in fieldTypeList
-  MODULE PROCEDURE ${ft.name}$_DEV_ALLOCATE_HST
-#:endfor
-END INTERFACE
-
 #:if defined('USE_BUDDY_MALLOC')
 INTERFACE DEV_ALLOCATE_DIM
 #:for ft in fieldTypeList
@@ -65,22 +59,6 @@ CONTAINS
 
 #:if defined('USE_BUDDY_MALLOC') or defined('WITH_HIC')
 
-SUBROUTINE ${ft.name}$_DEV_ALLOCATE_HST (DEV, HST, MAP_DEVPTR)
-
-${ft.type}$, POINTER :: DEV(${ft.shape}$)
-${ft.type}$, POINTER :: HST(${ft.shape}$)
-LOGICAL, INTENT(IN) :: MAP_DEVPTR
-
-INTEGER :: ILBOUNDS (${ft.rank}$)
-INTEGER :: IUBOUNDS (${ft.rank}$)
-
-ILBOUNDS = LBOUND (HST)
-IUBOUNDS = UBOUND (HST)
-
-CALL ${ft.name}$_DEV_ALLOCATE_DIM (DEV, UBOUNDS=IUBOUNDS, LBOUNDS=ILBOUNDS, MAP_DEVPTR=MAP_DEVPTR)
-
-END SUBROUTINE ${ft.name}$_DEV_ALLOCATE_HST
-
 SUBROUTINE ${ft.name}$_DEV_ALLOCATE_DIM (DEV, UBOUNDS, LBOUNDS, MAP_DEVPTR)
 
 USE FIELD_STATISTICS_MODULE
@@ -167,27 +145,6 @@ END SUBROUTINE ${ft.name}$_DEV_DEALLOCATE
 
 #:else
 
-SUBROUTINE ${ft.name}$_DEV_ALLOCATE_HST (DEV, HST, MAP_DEVPTR)
-
-USE FIELD_STATISTICS_MODULE
-
-${ft.type}$, POINTER :: DEV(${ft.shape}$)
-${ft.type}$, POINTER :: HST(${ft.shape}$)
-LOGICAL, INTENT(IN) :: MAP_DEVPTR
-
-#if __INTEL_COMPILER == 1800 && __INTEL_COMPILER_UPDATE == 5
-! Bug with Intel 18.0.5.274
-ALLOCATE (DEV (${ ', '.join (map (lambda i: 'LBOUND (HST, ' + str (i) + '):UBOUND (HST,' + str (i) + ')', range (1, ft.rank+1))) }$))
-#else
-ALLOCATE (DEV, MOLD=HST)
-#endif
-
-$:offload_macros.create(symbols=['DEV',])
-
-IF (FIELD_STATISTICS_ENABLE) CALL FIELD_STATISTICS_DEVICE_ALLOCATE (SIZE (DEV, KIND=JPIB) * INT (KIND (DEV), KIND=JPIB))
-
-END SUBROUTINE ${ft.name}$_DEV_ALLOCATE_HST
-
 SUBROUTINE ${ft.name}$_DEV_DEALLOCATE (DEV, MAP_DEVPTR)
 
 USE FIELD_STATISTICS_MODULE