! $Id$
!
! Earth System Modeling Framework
! Copyright (c) 2002-2024, University Corporation for Atmospheric Research,
! Massachusetts Institute of Technology, Geophysical Fluid Dynamics
! Laboratory, University of Michigan, National Centers for Environmental
! Prediction, Los Alamos National Laboratory, Argonne National Laboratory,
! NASA Goddard Space Flight Center.
! Licensed under the University of Illinois-NCSA License.
!
!==============================================================================

!==============================================================================
!ESMF_MULTI_PROC_EXAMPLE        String used by test script to count examples.
!==============================================================================

program ESMF_ArrayEx
#include "ESMF.h"

  use ESMF
  use ESMF_TestMod
  
  implicit none
  
  ! local variables
  integer:: rc, i, j, k, petCount, localDeCount, ssiLocalDeCount
  integer:: localPet, localDe
  type(ESMF_VM):: vm
  type(ESMF_DELayout):: delayout
  type(ESMF_DistGrid):: distgrid, distgrid3D, distgrid2D, distgrid1D
  type(ESMF_ArraySpec):: arrayspec
  type(ESMF_Array):: array, array1, array2, array1D, array2D, array3D
  type(ESMF_Array):: arrayTracer, arrayScalar, arrayMigrated
  type(ESMF_LocalArray), allocatable:: larrayList(:)
  type(ESMF_LocalArray), allocatable:: larrayList1(:), larrayList2(:)
  real(ESMF_KIND_R8), pointer:: myFarray(:,:)
  real(ESMF_KIND_R8), pointer:: myFarray1D(:), myFarray3D(:,:,:)
  real(ESMF_KIND_R8), pointer:: myFarray2D(:,:)
  real(ESMF_KIND_R8):: dummySum
  type(ESMF_Index_Flag):: indexflag
  integer, allocatable:: minIndex(:,:), maxIndex(:,:), regDecomp(:,:)
  integer, allocatable:: connectionList(:,:)
  integer, allocatable:: arrayToDistGridMap(:)
  integer, allocatable:: localDeToDeMap(:)
  integer, allocatable:: exclusiveLBound(:,:), exclusiveUBound(:,:)
  integer, allocatable:: totalLWidth(:,:), totalUWidth(:,:)
  integer, allocatable:: totalLBound(:,:), totalUBound(:,:)
!  integer, allocatable:: totalElementMask(:,:)
  integer, allocatable:: computationalLWidth(:,:), computationalUWidth(:,:)
  integer, allocatable:: computationalLBound(:,:), computationalUBound(:,:)
!  integer, allocatable:: haloLDepth(:), haloUDepth(:)
!  type(ESMF_RouteHandle):: haloHandle, haloHandle2

  logical :: ssiSharedMemoryEnabled

  character(ESMF_MAXSTR) :: testname
  character(ESMF_MAXSTR) :: failMsg


  ! result code
  integer :: finalrc, result
  
   finalrc = ESMF_SUCCESS
!-------------------------------------------------------------------------
!-------------------------------------------------------------------------

  write(failMsg, *) "Example failure"
  write(testname, *) "Example ESMF_ArrayEx"

!-------------------------------------------------------------------------
!-------------------------------------------------------------------------

  call ESMF_Initialize(vm=vm, defaultlogfilename="ArrayEx.Log", &
                    logkindflag=ESMF_LOGKIND_MULTI, rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
  call ESMF_VMGet(vm, localPet=localPet, petCount=petCount, rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
  
  if (petCount /= 4) then
    finalrc = ESMF_FAILURE
    goto 10
  endif

!===============================================================================

!BOE
! \subsubsection{Create Array with automatic memory allocation}
!
! In the examples of the previous sections the user provided memory allocations
! for each of the DE-local regions for an Array object. The user was able to 
! use any of the Fortran methods to allocate memory, or go through
! the {\tt ESMF\_LocalArray} interfaces to obtain memory allocations before
! passing them into ArrayCreate(). Alternatively ESMF offers methods that 
! handle Array memory allocations inside the library.
! 
! As before, to create an {\tt ESMF\_Array} object an {\tt ESMF\_DistGrid}
! must be created. The DistGrid object holds information about the entire 
! index space and how it is decomposed into DE-local exclusive regions. The 
! following line of code creates a DistGrid for a 5x5 global index space that 
! is decomposed into 2 x 3 = 6 DEs.
!EOE
!BOC
  distgrid = ESMF_DistGridCreate(minIndex=(/1,1/), maxIndex=(/5,5/), &
    regDecomp=(/2,3/), rc=rc)
!EOC  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! The following is a representation of the index space and its decomposition into
! DEs. Each asterisk (*) represents a single element.
!
! \begin{verbatim}
! 
!  +---------------------------------------> 2nd dimension
!  |  (1,1)
!  |    +-----------+-----------+------+
!  |    | DE 0      | DE 2      | DE 4 |
!  |    |           |           |      |
!  |    |  *    *   |  *    *   |  *   |
!  |    |           |           |      |
!  |    |  *    *   |  *    *   |  *   |
!  |    |           |           |      |
!  |    |  *    *   |  *    *   |  *   |
!  |    +-----------+-----------+------+
!  |    |           |           |      |
!  |    | DE 1      | DE 3      | DE 5 |
!  |    |           |           |      |
!  |    |  *    *   |  *    *   |  *   |
!  |    |           |           |      |
!  |    |  *    *   |  *    *   |  *   |
!  |    +-----------+-----------+------+
!  |                                 (5,5)
!  v 
! 1st dimension
!
! \end{verbatim}
!
! Besides the DistGrid it is the {\em type, kind} and {\em rank} information,
! "tkr" for short, that is required to create an Array object. It turns out that
! the rank of the Array object is fully determined by the DistGrid and other 
! (optional) arguments passed into ArrayCreate(), so that explicit 
! specification of the Array rank is redundant.
!
! The simplest way to supply the type and kind information of the Array is
! directly through the {\tt typekind} argument. Here a double precision Array
! is created on the previously created DistGrid. Since no other arguments are
! specified that could alter the rank of the Array it becomes equal to the 
! dimCount of the DistGrid, i.e a 2D Array is created on top of the DistGrid.
!EOE
!BOC
  array = ESMF_ArrayCreate(typekind=ESMF_TYPEKIND_R8, distgrid=distgrid, rc=rc)
!EOC  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! The different methods on how an Array object is created have no effect on
! the use of {\tt ESMF\_ArrayDestroy()}.
!EOE
!BOC
  call ESMF_ArrayDestroy(array, rc=rc)
!EOC  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! Alternatively the same Array can be created specifying the "tkr" information
! in form of an ArraySpec variable. The ArraySpec explicitly contains the 
! Array rank and thus results in an over specification on the ArrayCreate()
! interface. ESMF checks all input information for consistency and returns 
! appropriate error codes in case any inconsistencies are found.
!EOE
!BOC
  call ESMF_ArraySpecSet(arrayspec, typekind=ESMF_TYPEKIND_R8, rank=2, rc=rc)
!EOC  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOC
  array = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid, rc=rc)
!EOC  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!  call ESMF_ArrayPrint(array, rc=rc)
!  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!BOE
! The Array object created by the above call is an ESMF distributed 
! object. As such it must follow the ESMF convention that requires that 
! the call to {\tt ESMF\_ArrayCreate()} must be issued in unison by all 
! PETs of the current context.
!EOE

!===============================================================================

!BOE
! \subsubsection{Native language memory access}
! \label{Array_native_language_localde}
!
! There are two different methods by which the user can access the data held 
! inside an ESMF Array object. The first method provides direct access to a
! native language array object. Specifically, the {\tt farrayPtr} argument
! returned by {\tt ESMF\_ArrayGet()} is a Fortran array pointer that can be
! used to access the PET-local data inside the Array object.
!
! Many applications work in the 1 DE per PET mode, with exactly one DE on 
! every PET. Accessing the Array memory on each PET for this situation is 
! especially simple as is shown in section \ref{Array_from_native_1_to_1}.
! However, the Array class is not restricted to the special 1 DE per PET case,
! but supports multiple separate memory allocations on each PET.
! The number of such PET-local allocations is given by the {\tt localDeCount},
! i.e. there is one memory allocation for every DE that is associated with the
! local PET.
!
! Access to a specific local memory allocation of an Array object is still
! accomplished by returning the {\tt farrayPtr} argument. However, for
! $ localDeCount > 1 $ the formally optional {\tt localDe} argument to
! {\tt ESMF\_ArrayGet()} turns into a practically required argument. While
! in general the {\tt localDe} in ESMF is simply a local index variable that 
! enumerates the DEs that are associated with the local PET (e.g. see section
! \ref{DELayout_general_mapping}), the bounds of this index variable are 
! strictly defined as {\tt [0,...,localDeCount-1]} when it is used as an 
! input argument. The following code demonstrates this.
!
! First query the Array for {\tt localDeCount}. This number may be different
! on each PET and indicates how many DEs are mapped against the local PET.
!EOE
!BOC
  call ESMF_ArrayGet(array, localDeCount=localDeCount, rc=rc)
!EOC  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! Looping the {\tt localDe} index variable from 0 to {\tt localDeCount-1} allows
! access to each of the local memory allocations of the Array object:
!EOE
!BOC
  do localDe=0, localDeCount-1
    call ESMF_ArrayGet(array, farrayPtr=myFarray, localDe=localDe, rc=rc)
!EOC
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOC
    ! use myFarray to access local DE data
  enddo
!EOC
!
!BOE
! The second method to access the memory allocations in an Array object is to
! go through the ESMF LocalArray object. To this end the Array is queried
! for a list of PET-local LocalArray objects. The LocalArray objects in the list
! correspond to the DEs on the local PET. Here the {\tt localDe} argument is
! solely a user level index variable, and in principle the lower bound can be 
! chosen freely. However, for better alignment with the previous case (where 
! {\tt localDe} served as an input argument to an ESMF method) the following
! example again fixes the lower bound at zero.
!EOE
!BOC
  allocate(larrayList(0:localDeCount-1))
  call ESMF_ArrayGet(array, localarrayList=larrayList, rc=rc)
!EOC  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOC
  do localDe=0, localDeCount-1
    call ESMF_LocalArrayGet(larrayList(localDe), myFarray, &
       datacopyflag=ESMF_DATACOPY_REFERENCE, rc=rc)
!EOC
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOC
    ! use myFarray to access local DE data
  enddo
!EOC
!BOE
! See section \ref{Array:LocalArray} for more on LocalArray usage in Array. 
! In most cases memory access through a LocalArray list is less convenient than
! the direct {\tt farrayPtr} method because it adds an extra object level 
! between the ESMF Array and the native language array. 
!EOE

!===============================================================================

!BOE
! \subsubsection{Regions and default bounds}
! \label{Array_regions_and_default_bounds}
!
! Each {\tt ESMF\_Array} object is decomposed into DEs as specified by the
! associated {\tt ESMF\_DistGrid} object. Each piece of this decomposition, i.e.
! each DE, holds a chunk of the Array data in its own local piece of memory.
! The details of the Array decomposition are described in the following 
! paragraphs.
!
! At the center of the Array decomposition is the {\tt ESMF\_DistGrid} class.
! The DistGrid object specified during Array creation contains three essential
! pieces of information:
! \begin{itemize}
! \item The extent and topology of the global domain covered by the Array object
!       in terms of indexed elements. The total extent may be a composition of
!       smaller logically rectangular (LR) domain pieces called tiles.
! \item The decomposition of the entire domain into "element exclusive" DE-local
!       LR chunks. {\em Element exclusive} means that there is no element overlap
!       between DE-local chunks. This, however, does not exclude degeneracies 
!       on edge boundaries for certain topologies (e.g. bipolar).
! \item The layout of DEs over the available PETs and thus the distribution of
!       the Array data.
! \end{itemize}
!
! Each element of an Array is associated with a {\em single} DE. The union of
! elements associated with a DE, as defined by the DistGrid above, corresponds
! to a LR chunk of index space, called the {\em exclusive region} of the DE.
!
! There is a hierarchy of four regions that can be identified for each DE in an
! Array object. Their definition and relationship to each other is as follows:
! \begin{itemize}
! \item {\em Interior Region}: Region that only contains local elements that are
!       {\em not} mapped into the halo of any other DE. The shape and size of 
!       this region for a particular DE depends non-locally on the halos defined
!       by other DEs and may change during computation as halo operations are
!       precomputed and released. Knowledge of the interior elements may be used
!       to improve performance by overlapping communications with ongoing 
!       computation for a DE.
! \item {\em Exclusive Region}: Elements for which a DE claims exclusive
!       ownership. Practically this means that the DE will be the sole source
!       for these elements in halo and reduce operations. There are exceptions
!       to this in some topologies. The exclusive region includes all elements
!       of the interior region.
! \item {\em Computational Region}: Region that can be set arbitrarily within
!       the bounds of the total region (defined next). The typical use of the
!       computation region is to define bounds that only include elements that
!       are updated by a DE-local computation kernel. The computational region
!       does not need to include all exclusive elements and it may also contain
!       elements that lie outside the exclusive region.
! \item {\em Total (Memory) Region}: Total of all DE-locally allocated elements.
!       The size and shape of the total memory region must accommodate the
!       union of exclusive and computational region but may contain 
!       additional elements. Elements outside the exclusive region may overlap
!       with the exclusive region of another DE which makes them potential 
!       receivers for Array halo operations. Elements outside the exclusive
!       region that do not overlap with the exclusive region of another DE
!       can be used to set boundary conditions and/or serve as extra memory 
!       padding.
! \end{itemize}
!
! \begin{verbatim}
!
!   +-totalLBound(:)----------------------------------+
!   |\                                                |
!   | \ <--- totalLWidth(:)                           |
!   |  \                                              |
!   |   +-computationalLBound(:)------------------+   |
!   |   |\                                        |   |
!   |   | \ <--- computationalLWidth(:)           |   |
!   |   |  \                                      |   |
!   |   |   +-exclusiveLBound(:)-------------+    |   |
!   |   |   |                                |    |   |
!   |   |   |     +------+      +-----+      |    |   |
!   |   |   |     |      |      |     |      |    |   |
!   |   |   |     |      +------+     |      |    |   |
!   |   |   |     | "Interior Region" |      |    |   |
!   |   |   |     +-----+             |      |    |   |
!   |   |   |           |             |      |    |   |
!   |   |   |           +-------------+      |    |   |
!   |   |   |                                |    |   |
!   |   |   | "Exclusive Region"             |    |   |
!   |   |   +-------------exclusiveUBound(:)-+    |   |
!   |   |                                     \   |   |
!   |   |           computationalUWidth(:) --> \  |   |
!   |   |                                       \ |   |
!   |   | "Computational Region"                 \|   |
!   |   +------------------computationalUBound(:)-+   |
!   |                                              \  | 
!   |                             totalUWidth(:) -> \ | 
!   | "Total Region"                                 \| 
!   +--------------------------------- totalUBound(:)-+
! \end{verbatim}
!
!
! With the following definitions:
! \begin{verbatim}
!
! computationalLWidth(:) = exclusiveLBound(:) - computationalLBound(:)
! computationalUWidth(:) = computationalUBound(:) - exclusiveUBound(:)
!
! \end{verbatim}
! and
! \begin{verbatim}
!
! totalLWidth(:) = exclusiveLBound(:) - totalLBound(:)
! totalUWidth(:) = totalUBound(:) - exclusiveUBound(:)
!
! \end{verbatim}
!
!
! The {\em exclusive region} is determined during Array creation by the 
! DistGrid argument. Optional arguments may be used to specify the 
! {\em computational region} when the Array is created, by default it will be
! set equal to the exclusive region. The {\em total region}, i.e. the actual
! memory allocation for each DE, is also determined during Array creation. When
! creating the Array object from existing Fortran arrays the total region is
! set equal to the memory provided by the Fortran arrays. Otherwise the 
! default is to allocate as much memory as is needed to accommodate the union
! of the DE-local exclusive and computational region. Finally it is also
! possible to use optional arguments to the ArrayCreate() call to specify the
! total region of the object explicitly.
!
! The {\tt ESMF\_ArrayCreate()} call checks that the input parameters are
! consistent and will result in an Array that fulfills all of the above 
! mentioned requirements for its DE-local regions.
!
! Once an Array object has been created the exclusive and total regions are
! fixed. The computational region, however, may be adjusted within the limits
! of the total region using the {\tt ArraySet()} call.
!
! The {\em interior region} is very different from the other regions in that
! it cannot be specified. The {\em interior region} for each DE is a {\em
! consequence} of the choices made for the other regions collectively across
! all DEs into which an Array object is decomposed. An Array object can be
! queried for its DE-local {\em interior regions} as to offer additional
! information to the user necessary to write more efficient code.
! %See section 
! %\ref{ArrayEx_interiorRegion}(not yet implemented) for more details.
!
! By default the bounds of each DE-local {\em total region} are defined as
! to put the start of the DE-local {\em exclusive region} at the "origin" of 
! the local index space, i.e. at {\tt (1, 1, ..., 1)}. With that definition the
! following loop will access each element of the DE-local memory segment for
! each PET-local DE of the Array object used in the previous sections and
! print its content.
!EOE
!
!BOC
  do localDe=0, localDeCount-1
    call ESMF_LocalArrayGet(larrayList(localDe), myFarray, &
       datacopyflag=ESMF_DATACOPY_REFERENCE, rc=rc)
    do i=1, size(myFarray, 1)
      do j=1, size(myFarray, 2)
        print *, "localPET=", localPet, " localDE=", &
            localDe, ": array(",i,",",j,")=", myFarray(i,j)
      enddo
    enddo
  enddo
!EOC

!===============================================================================

!BOE
! \subsubsection{Array bounds}
!
! The loop over Array elements at the end of the last section only works
! correctly because of the default definition of the {\em computational} and
! {\em total regions} used in the example. In general, without such specific
! knowledge about an Array object, it is necessary to use a more formal approach
! to access its regions with DE-local indices.
!
! The DE-local {\em exclusive region} takes a central role in the definition
! of Array bounds. Even as the {\em computational region} may adjust during 
! the course of execution the {\em exclusive region} remains unchanged.
! The {\em exclusive region} provides a unique reference frame
! for the index space of all Arrays associated with the same DistGrid.
!
! There is a choice between two indexing options that needs to be made during 
! Array creation. By default each DE-local exclusive region starts at 
! {\tt (1, 1, ..., 1)}. However, for some computational kernels it may be more
! convenient to choose the index bounds of the DE-local exclusive regions to 
! match the index space coordinates as they are defined in the corresponding
! DistGrid object. The second option is only available if the DistGrid object 
! does not contain any non-contiguous decompositions (such as cyclically
! decomposed dimensions).
!
! The following example code demonstrates the safe way of dereferencing the
! DE-local exclusive regions of the previously created {\tt array} object.
!
!
!BOC
  allocate(exclusiveUBound(2, 0:localDeCount-1))  ! dimCount=2
  allocate(exclusiveLBound(2, 0:localDeCount-1))  ! dimCount=2
  call ESMF_ArrayGet(array, indexflag=indexflag, &
    exclusiveLBound=exclusiveLBound, exclusiveUBound=exclusiveUBound, rc=rc)
  if (indexflag == ESMF_INDEX_DELOCAL) then
    ! this is the default
!    print *, "DE-local exclusive regions start at (1,1)"
    do localDe=0, localDeCount-1
      call ESMF_LocalArrayGet(larrayList(localDe), myFarray, &
          datacopyflag=ESMF_DATACOPY_REFERENCE, rc=rc)
      do i=1, exclusiveUBound(1, localDe)
        do j=1, exclusiveUBound(2, localDe)
!          print *, "DE-local exclusive region for localDE=", localDe, &
!            ": array(",i,",",j,")=", myFarray(i,j)
        enddo
      enddo
    enddo
  else if (indexflag == ESMF_INDEX_GLOBAL) then
    ! only if set during ESMF_ArrayCreate()
!    print *, "DE-local exclusive regions of this Array have global bounds"
    do localDe=0, localDeCount-1
      call ESMF_LocalArrayGet(larrayList(localDe), myFarray, &
         datacopyflag=ESMF_DATACOPY_REFERENCE, rc=rc)
      do i=exclusiveLBound(1, localDe), exclusiveUBound(1, localDe)
        do j=exclusiveLBound(2, localDe), exclusiveUBound(2, localDe)
!          print *, "DE-local exclusive region for localDE=", localDe, &
!            ": array(",i,",",j,")=", myFarray(i,j)
        enddo
      enddo
    enddo
  endif
  call ESMF_ArrayDestroy(array, rc=rc) ! destroy the array object
!EOC  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!BOE
! Obviously the second branch of this simple code will work for either case, 
! however, if a complex computational kernel was written assuming 
! {\tt ESMF\_INDEX\_DELOCAL} type bounds the second branch would simply be 
! used to indicate the problem and bail out.
!
! The advantage of the {\tt ESMF\_INDEX\_GLOBAL} index option is that
! the Array bounds directly contain information on where the DE-local
! Array piece is located in a global index space sense. When the
! {\tt ESMF\_INDEX\_DELOCAL} option is used the correspondence between local
! and global index space must be made by querying the associated DistGrid for
! the DE-local {\tt indexList} arguments.
!EOE

!===============================================================================

!BOE
! \subsubsection{Computational region and extra elements for halo or padding}
! \label{Array:padding}
!
! In the previous examples the computational region of {\tt array} was chosen 
! by default to be identical to the exclusive region defined by the DistGrid
! argument during Array creation. In the following the same {\tt arrayspec} and
! {\tt distgrid} objects as before will be used to create an Array but now a 
! larger computational region shall be defined around each DE-local exclusive 
! region. Furthermore, extra space will be defined around the computational
! region of each DE to accommodate a halo and/or serve as memory padding.
!
! In this example the {\tt indexflag} argument is set to 
! {\tt ESMF\_INDEX\_GLOBAL} indicating that the bounds of the exclusive region
! correspond to the index space coordinates as they are defined by the DistGrid
! object.
!
! The same {\tt arrayspec} and {\tt distgrid} objects as before are used
! which also allows the reuse of the already allocated {\tt larrayList}
! variable.
!EOE
!BOC
  array = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid, &
    computationalLWidth=(/0,3/), computationalUWidth=(/1,1/), &
    totalLWidth=(/1,4/), totalUWidth=(/3,1/), &
    indexflag=ESMF_INDEX_GLOBAL, rc=rc)
!EOC  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!  call ESMF_ArrayPrint(array, rc=rc)
!  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! Obtain the {\tt larrayList} on every PET.
!EOE
!BOC
  allocate(localDeToDeMap(0:localDeCount-1))
  call ESMF_ArrayGet(array, localarrayList=larrayList, &
    localDeToDeMap=localDeToDeMap, rc=rc)
!EOC  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! The bounds of DE 1 for {\tt array} are shown in the following 
! diagram to illustrate the situation. Notice that the {\tt totalLWidth} and
! {\tt totalUWidth} arguments in the ArrayCreate() call define the total region 
! with respect to the exclusive region given for each DE by the {\tt distgrid} 
! argument.
! 
! \begin{verbatim}
!      +-(3,-3)---------------------------------+ 
!      |\                                       | 
!      | +-(4,-2)-+-(4,1)--------------------+--+ 
!      | |        |                          |  | 
!      | |        |                          |  | 
!      | |        |          DE 1            |  | 
!      | |        |                          |  | 
!      | |        |                          |  | 
!      | |        | Exclusive Region         |  | 
!      | |        +--------------------(5,2)-+  | 
!      | | Computational Region                 | 
!      | +-------------------------------(6,3)--+ 
!      |                                        | 
!      | Total Region                           | 
!      +---------------------------------(8,3)--+ 
! \end{verbatim}
!
! When working with this {\tt array} it is possible for the computational
! kernel to overstep the exclusive region for both read/write access 
! (computational region) and potentially read-only access into the total region
! outside of the computational region, if a halo operation provides valid 
! entries for these elements. 
!
! The Array object can be queried for absolute {\em bounds}
!EOE
!BOC
  allocate(computationalLBound(2, 0:localDeCount-1))  ! dimCount=2
  allocate(computationalUBound(2, 0:localDeCount-1))  ! dimCount=2
  allocate(totalLBound(2, 0:localDeCount-1))          ! dimCount=2
  allocate(totalUBound(2, 0:localDeCount-1))          ! dimCount=2
  call ESMF_ArrayGet(array, exclusiveLBound=exclusiveLBound, &
    exclusiveUBound=exclusiveUBound, &
    computationalLBound=computationalLBound, &
    computationalUBound=computationalUBound, &
    totalLBound=totalLBound, &
    totalUBound=totalUBound, rc=rc)
!EOC  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! or for the relative {\em widths}.
!EOE
!BOC
  allocate(computationalLWidth(2, 0:localDeCount-1))  ! dimCount=2
  allocate(computationalUWidth(2, 0:localDeCount-1))  ! dimCount=2
  allocate(totalLWidth(2, 0:localDeCount-1))          ! dimCount=2
  allocate(totalUWidth(2, 0:localDeCount-1))          ! dimCount=2
  call ESMF_ArrayGet(array, computationalLWidth=computationalLWidth, &
    computationalUWidth=computationalUWidth, totalLWidth=totalLWidth, &
    totalUWidth=totalUWidth, rc=rc)
!EOC
!BOE
! Either way the dereferencing of Array data is centered around the DE-local
! exclusive region:
!EOE
!BOC
  do localDe=0, localDeCount-1
    call ESMF_LocalArrayGet(larrayList(localDe), myFarray, &
       datacopyflag=ESMF_DATACOPY_REFERENCE, rc=rc)
    ! initialize the DE-local array
    myFarray = 0.1d0 * localDeToDeMap(localDe)
    ! first time through the total region of array    
!    print *, "myFarray bounds for DE=", localDeToDeMap(localDe), &
!      lbound(myFarray),  ubound(myFarray)
    do j=exclusiveLBound(2, localDe), exclusiveUBound(2, localDe)
      do i=exclusiveLBound(1, localDe), exclusiveUBound(1, localDe)
!        print *, "Excl region DE=", localDeToDeMap(localDe), &
!        ": array(",i,",",j,")=",  myFarray(i,j)
      enddo
    enddo
    do j=computationalLBound(2, localDe), computationalUBound(2, localDe)
      do i=computationalLBound(1, localDe), computationalUBound(1, localDe)
!        print *, "Excl region DE=", localDeToDeMap(localDe), &
!        ": array(",i,",",j,")=", myFarray(i,j)
      enddo
    enddo
    do j=totalLBound(2, localDe), totalUBound(2, localDe)
      do i=totalLBound(1, localDe), totalUBound(1, localDe)
!        print *, "Total region DE=", localDeToDeMap(localDe), &
!        ": array(",i,",",j,")=", myFarray(i,j)
      enddo
    enddo

    ! second time through the total region of array    
    do j=exclusiveLBound(2, localDe)-totalLWidth(2, localDe), &
      exclusiveUBound(2, localDe)+totalUWidth(2, localDe)
      do i=exclusiveLBound(1, localDe)-totalLWidth(1, localDe), &
        exclusiveUBound(1, localDe)+totalUWidth(1, localDe)
!        print *, "Excl region DE=", localDeToDeMap(localDe), &
!        ": array(",i,",",j,")=", myFarray(i,j)
      enddo
    enddo
  enddo
!EOC

  deallocate(larrayList)
  deallocate(localDeToDeMap)
  call ESMF_ArrayDestroy(array, rc=rc) ! finally destroy the array object
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
  
  
!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
!!!! UNTIL FURTHER IMPLEMENTATION SKIP SECTIONS OF THIS EXAMPLE >>>>>>>>>>>>>>>>
#ifdef NOSKIP
!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

!BOEI
! 
! The index space topology of this example is very simple. The DistGrid is
! defined by a single LR tile and does not contain any extra connections.
! Consequently,
! DEs that are located at the edge of the index space may contain DE-local 
! computational and total regions that reach beyond the index space as it is
! defined by 
! the DistGrid. For example DE 1's computational and total region, as shown 
! in the diagram above, contain extra elements that are not covered by the
! index space described by the DistGrid object. 
!
! The situation is different for interior DEs, or when the widths specified
! during Array creation are asymmetric. For the current example DE 3, for
! instance,
! only contains elements in its total and computational region that lie within
! the DistGrid index space. Elements in the total region that are outside of a 
! DE's exclusive region are then redundant with another DE's exclusive elements. 
! Redundancies in the computational region may be enforced or monitored by
! special Array communication calls. Total region elements that are outside the 
! computational region may be part of a halo operation and may be updated with 
! the value of the corresponding exclusive element in another DE using halo 
! communication methods.
!
! Computational elements that are outside the DistGrid index space are kept for 
! convenience, simplifying the formulation of the computational kernel. 
! Naturally, redundancy cannot be checked or enforced for these elements. 
! Similarly, halo elements that do not correspond to elements within the index space
! of the DistGrid will not be updated during halo communications. It is up to
! the application writer to utilize the provided elements as is appropriate for the
! computational algorithm. The Array class offers mask type information 
! through the {\tt ESMF\_ArrayGet()} method in order to assist the user in 
! dealing with elements that are not part of the Array index space as it is 
! defined in the corresponding DistGrid object.
! %Please see section 
! %\ref{ArrayEx_interiorRegion} for details.
!EOEI

!===============================================================================

!BOEI
! \subsubsection{Interior region and Array's total element mask}
! \label{ArrayEx_interiorRegion}
!
! The {\em Array total element mask} is a native Fortran integer array of
! the same rank as the ESMF Array object. The array must be allocated and 
! deallocated by the user and is filled by the ArrayGet() method with the
! mask information. (The Array object does not store the mask information
! internally (to save memory) but generates the information on the fly when
! inquired.)
!
! The array variable used as mask argument must be allocated to 
! accommodate the total region for the inquired DE. The bounds must
! be chosen to match the totalLBound(:) and totalUBound(:) parameters of the
! Array.
!
! For the {\tt array} object created in the previous section the total element mask
! for DE 1 would need to be allocated as
!EOEI
!BOCI
  allocate(totalElementMask(3:8, -3:3))              ! rank=2
!EOCI
!BOEI
! Then, assuming that DE 1 was localDE 1 on the PET issuing the following
! call the mask can be obtained in the following manner.
!EOEI
!BOCI
  call ESMF_ArrayGet(array, localDe=1, totalElementMask=totalElementMask, rc=rc)
!EOCI
!BOEI
! Now the {\tt totalElementMask} variable contains the mask for the total
! region for DE 1 for {\tt array}. For the example the mask will contain the
! following values:
!
! \begin{verbatim}
!
!  +-------------------------------------------> 2nd dimension
!  |  
!  |
!  |    (3,-3)
!  |      +-----------------------------+
!  |      | -1  -1  -1  -1   0   0   2  |
!  |      |    +-----------+-------+----+
!  |      | -1 |-1  -1  -1 |-2  -2 | 3  |
!  |      |    |           |       |    |
!  |      | -1 |-1  -1  -1 |-2  -2 | 3  |
!  |      |    |           +-------+    |
!  |      | -1 |-1  -1  -1  -1  -1  -1  |
!  |      |    +------------------------+
!  |      | -1  -1  -1  -1  -1  -1  -1  |
!  |      |                             |
!  |      | -1  -1  -1  -1  -1  -1  -1  |
!  |      +-----------------------------+
!  |                                  (8,3)
!  |
!  v 1st dimension
!
! \end{verbatim}
!
! The inner most rectangle corresponds to the exclusive region on DE 1. The
! value of "-2" in an exclusive element indicates that other DEs of the Array
! have elements in their total region that correspond to this DE's exclusive element.
! A value of "-3" for an exclusive element indicates that there exists a redundancy
! with another DE's exclusive region and a value of "-4" indicates redundancy 
! and dependency between DEs. Finally, a element value of "-1" in the exclusive
! region indicates that there are no other DEs in the Array that depend on this
! exclusive element. The union of exclusive elements with mask value "-1" define the 
! {\em absolute interior region} of the DE. In this example DE 1 does not 
! contain any absolute interior elements.
!
! The next larger rectangle in the above diagram corresponds to the 
! computational region of DE 1 and the entire array extent is that of this DE's
! total region. A element mask value of "-1" in these regions indicates that 
! there are no elements in any of the DEs' exclusive regions that corresponds to 
! this element, i.e. the element lies outside the index space covered by DistGrid.
!
! Mask values $>= 0$ in the computational and total regions indicate the DE 
! which contains the corresponding element in its exclusive region. Thus a element 
! with mask value $>= 0$ lies within the DistGrid index space.
!
! The query call used above was only able to return information based on the
! information stored in the Array object and the underlying DistGrid. However,
! it is often more valuable to obtain the information stored in a total element
! mask specifically for a particular Array communication pattern or a whole
! set thereof. The following lines of code demonstrate this aspect for one of 
! the Array halo operations used in the previous section.
!EOEI
!BOCI
  call ESMF_ArrayHaloStore(array, haloLDepth=(/0,0/), haloUDepth=(/0,1/), &
    routehandle=haloHandle2, rc=rc)
!EOCI
!BOEI
! The same {\tt totalElementMask} variable may be used to obtain the specific total
! element mask. Only difference is that the {\tt routehandle} must be provided
! as an additional input parameter. Since specific total element masks may be 
! required as an "OR" over a whole set of communication operations the optional
! input parameter is defined as a list of RouteHandles.
!EOEI
!BOCI
  call ESMF_ArrayGet(array, routehandlelist=(/haloHandle2/), &
    localDe=1, totalElementMask=totalElementMask, rc=rc)
!EOCI
!BOEI
! Now {\tt totalElementMask} holds the total element mask for {\tt array} specifically
! for the Route referenced by {\tt haloHandle2}. For DE1 the output will look
! as follows:
! \begin{verbatim}
!
!  +-------------------------------------------> 2nd dimension
!  |  
!  |
!  |    (3,-3)
!  |      +-----------------------------+
!  |      | -1  -1  -1  -1  -1  -1  -1  |
!  |      |    +-----------+-------+----+
!  |      | -1 |-1  -1  -1 |-1  -1 | 3  |
!  |      |    |           |       |    |
!  |      | -1 |-1  -1  -1 |-1  -1 | 3  |
!  |      |    |           +-------+    |
!  |      | -1 |-1  -1  -1  -1  -1  -1  |
!  |      |    +------------------------+
!  |      | -1  -1  -1  -1  -1  -1  -1  |
!  |      |                             |
!  |      | -1  -1  -1  -1  -1  -1  -1  |
!  |      +-----------------------------+
!  |                                  (8,3)
!  |
!  v 1st dimension
!
! \end{verbatim}
!
! The meaning of the values stored in the total element mask for a specific 
! query call are now related to the specified operations. The "-1" in the 
! exclusive region (inner most rectangle) indicates that the operation
! referenced by {\tt haloHandle2} requires none of the exclusive elements of DE 1.
! Thus all exclusive elements of DE 1 can be considered interior elements with 
! respect to {\tt haloHandle2}. Consequently, if a non-blocking HaloRun() were
! to be executed it would be safe to read from and write to all of DE 1's
! exclusive elements.
!
! For the next larger rectangle, the computational region, {\tt totalElementMask}
! indicates that there are only two elements that will be updated with values from 
! exclusive elements of DE 3 during execution of {\tt haloHandle2}, all other elements
! are unaffected by this communication pattern.
!
! Finally, none of the points in DE 1's total region outside of the 
! computational region are affected by {\tt haloHandle2}.
!
! After usage it is the user's responsibility to clean up.
!EOEI
!BOCI
  deallocate(totalElementMask)
  call ESMF_RouteHandleRelease(routehandle=haloHandle2, rc=rc)
  call ESMF_ArrayDestroy(array, rc=rc) ! destroy array object
!EOCI  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOEI
! 
!
!
! \subsubsection{Total region, memory padding and changing computational region}
!
! The default total region during Array creation is chosen as to exactly
! accommodate the computational region. Once an Array object has been created 
! the total region for each DE is fixed and cannot be changed. The computational
! width, however, may be changed using the ArraySet() call. Changes are allowed
! within the limits set by the fixed total region.
!
! The following code illustrates how an Array can be created to accommodate the
! largest total region that is expected to occur during the compute cycle. The 
! example also shows how the computational region may change during the compute
! cycle. Again the same {\tt arrayspec} and {\tt distgrid} arguments as before 
! are used.
!EOEI
!BOCI
  array = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid, &
    totalLWidth=(/3,3/), totalUWidth=(/3,3/), rc=rc)
!EOCI  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOEI
! This {\tt array} has DE-local total regions that are three elements wider in 
! each direction than the corresponding exclusive region. The default
! computational region for each DE is equal to the DE's exclusive region. The
! extra elements between exclusive and total region are available for halos or
! can be used to expand the computational region of the Array.
!
! The following compute loop first executes a halo update which by default
! will attempt to halo all elements that are outside of the computational region.
! Then the a three step cycle is entered which sets the computational region to
! 2, 1 and 0 elements around the exclusive region and each time calls a 
! computational kernel that will access elements from (i-1,j-1) to (i+1,j+1). 
! After the computational width has reached 0 a new halo updated needs to be
! performed.
!EOEI
!BOCI
  do j=1, 10
    call ESMF_ArrayHalo(array, rc=rc)
    do k=2, 0, -1
      call ESMF_ArraySet(array, &
        computationalLWidth=(/k,k/), computationalUWidth=(/k,k/), rc=rc)
      ! call second_order_kernel(array)  ! will access (i-1) and (i+1) elements
    enddo
  enddo
!EOCI
!BOCI
  call ESMF_ArrayDestroy(array, rc=rc) ! finally destroy the array object
!EOCI  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOEI
!EOCI
!BOEI
! Sometimes it may be desirable to add extra memory padding around the DE-local
! data elements, e.g. to prevent false sharing on shared memory architectures. 
! For this the total region can be chosen larger than is needed by the 
! algorithm. The following lines of code recapture the example from above but
! now the total region is chosen with a 5 element wide rim around the exclusive
! region. Still only a 3 element wide halo is needed for the computational
! algorithm. By default the ArrayHalo() call will halo all extra elements, to keep
! the halo to 3 elements in each direction the haloLDepth and haloUDepth arguments
! must be used. With that a 5 - 3 = 2 element wide memory padding has been placed
! around the elements used by each DE.
!
!EOEI
!BOCI
  array = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid, &
    totalLWidth=(/5,5/), totalUWidth=(/5,5/), rc=rc)
!EOCI  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOCI
  do j=1, 10
    call ESMF_ArrayHalo(array, haloLDepth=(/3,3/), haloUDepth=(/3,3/), rc=rc)
    do k=2, 0, -1
      call ESMF_ArraySet(array, &
        computationalLWidth=(/k,k/), computationalUWidth=(/k,k/), rc=rc)
      ! call second_order_kernel(array)  ! will access (i-1) and (i+1) elements
    enddo
  enddo
!EOCI
!BOEI
! Done, {\tt array} and {\tt distgrid} can be destroyed.
!EOEI
!BOCI  
  call ESMF_ArrayDestroy(array, rc=rc) ! finally destroy the array object
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
  call ESMF_DistGridDestroy(distgrid, rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!EOCI  

#else
  ! need to clean up distgrid that was created several sections ago
  call ESMF_DistGridDestroy(distgrid, rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#endif
!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
  
!BOE
! \subsubsection{Create 1D and 3D Arrays}
!
! All previous examples were written for the 2D case. There is, however, no
! restriction within the Array or DistGrid class that limits the dimensionality
! of Array objects beyond the language-specific limitations (7D for Fortran).
!
! In order to create an {\tt n}-dimensional Array the rank indicated by both
! the {\tt arrayspec} and the {\tt distgrid} arguments specified during Array
! create must be equal to {\tt n}. A 1D Array of double precision real data
! hence requires the following {\tt arrayspec}.
!EOE  
!BOC
  call ESMF_ArraySpecSet(arrayspec, typekind=ESMF_TYPEKIND_R8, rank=1, rc=rc)
!EOC
!BOE
! The index space covered by the Array and the decomposition description is
! provided to the Array create method by the {\tt distgrid} argument. The index
! space in this example has 16 elements and covers the interval $[-10, 5]$. It is 
! decomposed into as many DEs as there are PETs in the current context.
!EOE
!BOC
  distgrid1D = ESMF_DistGridCreate(minIndex=(/-10/), maxIndex=(/5/), &
    regDecomp=(/petCount/), rc=rc)
!EOC
!BOE
! A 1D Array object with default regions can now be created.
!EOE
!BOC
  array1D = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid1D, rc=rc)
!EOC
  call ESMF_ArrayDestroy(array1D, rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
  call ESMF_DistGridDestroy(distgrid1D, rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! 
! The creation of a 3D Array proceeds analogous to the 1D case. The rank of the
! {\tt arrayspec} must be changed to 3
!EOE
!BOC
  call ESMF_ArraySpecSet(arrayspec, typekind=ESMF_TYPEKIND_R8, rank=3, rc=rc)
!EOC
!BOE
! and an appropriate 3D DistGrid object must be created
!EOE
!BOC
  distgrid3D = ESMF_DistGridCreate(minIndex=(/1,1,1/), &
    maxIndex=(/16,16,16/), regDecomp=(/4,4,4/), rc=rc)
!EOC
!BOE
! before an Array object can be created.
!EOE
!BOC
  array3D = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid3D, rc=rc)
!EOC
!BOE
! The {\tt distgrid3D} object decomposes the 3-dimensional index space into
! $4\times 4\times 4 = 64$ DEs. These DEs are laid out across the computational
! resources (PETs) of the current component according to a default DELayout that
! is created during the DistGrid create call. Notice that in the index space 
! proposal a DELayout does not have a sense of dimensionality. The DELayout
! function is simply to map DEs to PETs. The DistGrid maps chunks of index space
! against DEs and thus its rank is equal to the number of index space 
! dimensions.
!
! The previously defined DistGrid and the derived Array object decompose 
! the index space along all three dimension. It is, however, not a requirement
! that the decomposition be along all dimensions. An Array with the same 3D
! index space could as well be decomposed along just one or along two of the
! dimensions. The following example shows how for the same index space only the
! last two dimensions are decomposed while the first Array dimension has full
! extent on all DEs.
!EOE
!BOC
  call ESMF_ArrayDestroy(array3D, rc=rc)
  call ESMF_DistGridDestroy(distgrid3D, rc=rc)
  distgrid3D = ESMF_DistGridCreate(minIndex=(/1,1,1/), &
    maxIndex=(/16,16,16/), regDecomp=(/1,4,4/), rc=rc)
  array3D = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid3D, rc=rc)
!EOC

!===============================================================================

!BOE
! \subsubsection{Working with Arrays of different rank}
! Assume a computational kernel that involves the {\tt array3D} object as it was
! created at the end of the previous section. Assume further that the kernel 
! also involves a 2D Array on a 16x16 index space where each point (j,k) was
! interacting with each (i,j,k) column of the 3D Array. An efficient formulation
! would require that the decomposition of the 2D Array must match that of the 3D
! Array and further the DELayout be identical. The following code shows how this
! can be accomplished.
!EOE
!BOC
  call ESMF_DistGridGet(distgrid3D, delayout=delayout, rc=rc) ! get DELayout
  distgrid2D = ESMF_DistGridCreate(minIndex=(/1,1/), maxIndex=(/16,16/), &
    regDecomp=(/4,4/), delayout=delayout, rc=rc)
  call ESMF_ArraySpecSet(arrayspec, typekind=ESMF_TYPEKIND_R8, rank=2, rc=rc)
  array2D = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid2D, rc=rc)
!EOC
!BOE
! Now the following kernel is sure to work with {\tt array3D} and {\tt array2D}.
!EOE
!BOC
  call ESMF_DELayoutGet(delayout, localDeCount=localDeCount, rc=rc)
  allocate(larrayList1(0:localDeCount-1))
  call ESMF_ArrayGet(array3D, localarrayList=larrayList1, rc=rc)
  allocate(larrayList2(0:localDeCount-1))
  call ESMF_ArrayGet(array2D, localarrayList=larrayList2, rc=rc)
  do localDe=0, localDeCount-1
    call ESMF_LocalArrayGet(larrayList1(localDe), myFarray3D, &
      datacopyflag=ESMF_DATACOPY_REFERENCE, rc=rc)
    myFarray3D = 0.1d0 * localDe ! initialize
    call ESMF_LocalArrayGet(larrayList2(localDe), myFarray2D, &
      datacopyflag=ESMF_DATACOPY_REFERENCE, rc=rc)
    myFarray2D = 0.5d0 * localDe ! initialize
    do k=1, 4
      do j=1, 4
        dummySum = 0.d0
        do i=1, 16
          dummySum = dummySum + myFarray3D(i,j,k) ! sum up the (j,k) column
        enddo
        dummySum = dummySum * myFarray2D(j,k) ! multiply with local 2D element
!        print *, "dummySum(",j,k,")=",dummySum
      enddo
    enddo
  enddo
!EOC

!call ESMF_ArrayPrint(array3D)
!call ESMF_ArrayPrint(array2D)
  call ESMF_ArrayDestroy(array2D, rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
  call ESMF_DistGridDestroy(distgrid2D, rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
  call ESMF_ArrayDestroy(array3D, rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
  call ESMF_DistGridDestroy(distgrid3D, rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!===============================================================================

!BOE
! \subsubsection{Arrays with undistributed dimensions}
! \label{sec:array:usage:undistributed_dims}
!
! Except for the special Array create interface that implements a copy from
! an existing Array object all other Array create interfaces require the 
! specification of at least two arguments: {\tt farray} and {\tt distgrid},
! {\tt larrayList} and {\tt distgrid}, or {\tt arrayspec} and {\tt distgrid}.
! In all these cases both required arguments contain a sense of dimensionality.
! The relationship between these two arguments deserves extra attention.
!
! The first argument, {\tt farray}, {\tt larrayList} or {\tt arrayspec}, 
! determines the rank of the created Array object, i.e. the dimensionality
! of the actual data storage. The rank of a native language array, extracted 
! from an Array object, is equal to the rank specified by either of these
! arguments. So is the {\tt rank} that is returned by the {\tt ESMF\_ArrayGet()}
! call.
!
! The rank specification contained in the {\tt distgrid} argument, which is of 
! type {\tt ESMF\_DistGrid}, on the other hand has no effect on the
! rank of the Array. The {\tt dimCount} specified by the DistGrid object,
! which may be equal, greater or less than the Array rank, determines the 
! dimensionality of the {\em decomposition}.
!
! While there is no constraint between DistGrid {\tt dimCount} and Array
! {\tt rank}, there is an important relationship between the two, resulting in
! the concept of index space dimensionality. Array dimensions can be
! arbitrarily mapped against DistGrid dimension, rendering them {\em decomposed}
! dimensions. The index space dimensionality is equal to the number of 
! decomposed Array dimensions.
!
! \begin{sloppypar}
! Array dimensions that are not mapped to DistGrid dimensions are the 
! {\em undistributed} dimensions of the Array. They are not part
! of the index space. The mapping is specified during {\tt ESMF\_ArrayCreate()}
! via the {\tt distgridToArrayMap} argument. DistGrid dimensions that have
! not been associated with Array dimensions are {\em replicating} dimensions.
! The Array will be replicated across the DEs that lie along replication
! DistGrid dimensions.
! \end{sloppypar}
!
! Undistributed Array dimensions can be used to store multi-dimensional data for
! each Array index space element. One application of this is to store the 
! components of a vector quantity in a single Array. The same 2D {\tt distgrid}
! object as before will be used.
!EOE
!BOC
  distgrid = ESMF_DistGridCreate(minIndex=(/1,1/), maxIndex=(/5,5/), &
    regDecomp=(/2,3/), rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! The rank in the {\tt arrayspec} argument, however, must change from 2 to 3 in
! order to provide for the extra Array dimension.
!EOE
!BOC
  call ESMF_ArraySpecSet(arrayspec, typekind=ESMF_TYPEKIND_R8, rank=3, rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! During Array creation with extra dimension(s) it is necessary to specify the
! bounds of these undistributed dimension(s). This requires two additional
! arguments, {\tt undistLBound} and {\tt undistUBound}, which are vectors in 
! order to accommodate multiple undistributed dimensions. The other arguments
! remain unchanged and apply across all undistributed components. 
!
!EOE
!BOC
  array = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid, &
    totalLWidth=(/0,1/), totalUWidth=(/0,1/), &
    undistLBound=(/1/), undistUBound=(/2/), rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!EOC
    
!  call ESMF_ArrayPrint(array, rc=rc)
!  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
    
!BOE
! This will create {\tt array} with 2+1 dimensions. The 2D DistGrid is used
! to describe decomposition into DEs with 2 Array dimensions mapped to the 
! DistGrid dimensions resulting in a 2D index space. The extra Array dimension
! provides storage for multi component user data within the Array object.
!
! By default the {\tt distgrid} dimensions are associated
! with the first Array dimensions in sequence. For the example above this means
! that the first 2 Array dimensions are decomposed according to the provided 2D
! DistGrid. The 3rd Array dimension does not have an associated DistGrid
! dimension, rendering it an undistributed Array dimension.
!
! Native language access to an Array with undistributed dimensions is in
! principle the same as without extra dimensions.
!EOE
!BOC
  call ESMF_ArrayGet(array, localDeCount=localDeCount, rc=rc)
  allocate(larrayList(0:localDeCount-1))
  call ESMF_ArrayGet(array, localarrayList=larrayList, rc=rc)
!BOE
! The following loop shows how a Fortran pointer to the DE-local data chunks
! can be obtained and used to set data values in the exclusive regions. The
! {\tt myFarray3D} variable must be of rank 3 to match the Array rank of
! {\tt array}. However, variables such as {\tt exclusiveUBound} that store the
! information about the decomposition, remain to be allocated for the 2D 
! index space.
!EOE
!BOC
  call ESMF_ArrayGet(array, exclusiveLBound=exclusiveLBound, &
    exclusiveUBound=exclusiveUBound, rc=rc)
  do localDe=0, localDeCount-1
    call ESMF_LocalArrayGet(larrayList(localDe), myFarray3D, &
       datacopyflag=ESMF_DATACOPY_REFERENCE, rc=rc)
    myFarray3D = 0.0 ! initialize
    myFarray3D(exclusiveLBound(1,localDe):exclusiveUBound(1,localDe), &
      exclusiveLBound(2,localDe):exclusiveUBound(2,localDe), &
      1) = 5.1 ! dummy assignment
    myFarray3D(exclusiveLBound(1,localDe):exclusiveUBound(1,localDe), &
      exclusiveLBound(2,localDe):exclusiveUBound(2,localDe), &
      2) = 2.5 ! dummy assignment
  enddo
  deallocate(larrayList)
!EOC
!  call ESMF_ArrayPrint(array, rc=rc)
!  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! For some applications the default association rules between DistGrid and Array
! dimensions may not satisfy the user's needs. The optional {\tt distgridToArrayMap} 
! argument can be used during Array creation to explicitly specify the mapping 
! between DistGrid and Array dimensions. To demonstrate this the following lines
! of code reproduce the above example but with rearranged dimensions. Here the
! {\tt distgridToArrayMap} argument is a list with two elements corresponding to
! the DistGrid {\tt dimCount} of 2. The first element indicates which Array
! dimension the first DistGrid dimension is mapped against. Here the
! 1st DistGrid dimension maps against the 3rd Array dimension and the 2nd 
! DistGrid dimension maps against the 1st Array dimension. This leaves the 2nd
! Array dimension to be the extra and undistributed dimension in the resulting
! Array object.
!EOE
!BOC
  call ESMF_ArrayDestroy(array, rc=rc)
  array = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid, &
    distgridToArrayMap=(/3, 1/), totalLWidth=(/0,1/), totalUWidth=(/0,1/), &
    undistLBound=(/1/), undistUBound=(/2/), rc=rc)
!EOC
!BOE
! Operations on the Array object as a whole are unchanged by the different
! mapping of dimensions.
!
! When working with Arrays that contain explicitly mapped Array and DistGrid 
! dimensions it is critical to know the order in which the entries of
! {\em width} and {\em bound} arguments that are associated with distributed
! Array dimensions are specified. The size of these arguments is equal to the
! DistGrid {\tt dimCount}, because the maximum number of distributed Array
! dimensions is given by the dimensionality of the index space.
!
! The order of dimensions in these arguments, however, is {\em not} that of
! the associated DistGrid. Instead each entry corresponds to the distributed
! Array dimensions in sequence. In the example above the entries in 
! {\tt totalLWidth} and {\tt totalUWidth} correspond to Array dimensions 1 and
! 3 in this sequence. 
!
! The {\tt distgridToArrrayMap} argument optionally provided during Array create
! indicates how the DistGrid dimensions map to Array dimensions. The inverse
! mapping, i.e. Array to DistGrid dimensions, is just as important. The 
! {\tt ESMF\_ArrayGet()} call offers both mappings as {\tt distgridToArrrayMap}
! and {\tt arrayToDistGridMap}, respectively. The number of elements in 
! {\tt arrayToDistGridMap} is equal to the rank of the Array. Each element
! corresponds to an Array dimension and indicates the associated DistGrid
! dimension by an integer number. An entry of "0" in {\tt arrayToDistGridMap}
! indicates that the corresponding Array dimension is undistributed.
!
! Correct understanding about the association between Array and DistGrid
! dimensions becomes critical for correct data access into the Array.
!EOE
!BOC
  allocate(arrayToDistGridMap(3))  ! arrayRank = 3
  call ESMF_ArrayGet(array, arrayToDistGridMap=arrayToDistGridMap, &
    exclusiveLBound=exclusiveLBound, exclusiveUBound=exclusiveUBound, &
    localDeCount=localDeCount, rc=rc)  
  if (arrayToDistGridMap(2) /= 0) then   ! check if extra dimension at 
    ! expected index indicate problem and bail out
  endif
  ! obtain larrayList for local DEs
  allocate(larrayList(0:localDeCount-1))
  call ESMF_ArrayGet(array, localarrayList=larrayList, rc=rc)
  do localDe=0, localDeCount-1
    call ESMF_LocalArrayGet(larrayList(localDe), myFarray3D, &
       datacopyflag=ESMF_DATACOPY_REFERENCE, rc=rc)
    myFarray3D(exclusiveLBound(1,localDe):exclusiveUBound(1,localDe), &
      1, exclusiveLBound(2,localDe):exclusiveUBound(2, &
      localDe)) = 10.5 !dummy assignment
    myFarray3D(exclusiveLBound(1,localDe):exclusiveUBound(1,localDe), &
      2, exclusiveLBound(2,localDe):exclusiveUBound(2, &
      localDe)) = 23.3 !dummy assignment
  enddo
  deallocate(exclusiveLBound, exclusiveUBound)
  deallocate(arrayToDistGridMap)
  deallocate(larrayList)
  call ESMF_ArrayDestroy(array, rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!EOC

!===============================================================================

!BOE
! \subsubsection{Arrays with replicated dimensions}
! \label{sec:array:usage:replicated_dims}
!
! Thus far most examples demonstrated cases where the DistGrid {\tt dimCount}
! was equal to the Array {\tt rank}. The previous section introduced the
! concept of Array {\em tensor} dimensions when {\tt dimCount < rank}. In this
! section {\tt dimCount} and {\tt rank} are assumed completely unconstrained and
! the relationship to {\tt distgridToArrayMap} and {\tt arrayToDistGridMap} will
! be discussed.
!
! The Array class allows completely arbitrary mapping between Array and
! DistGrid dimensions. Most cases considered in the previous sections used
! the default mapping which assigns the DistGrid dimensions in sequence to the
! lower Array dimensions. Extra Array dimensions, if present, are considered
! non-distributed tensor dimensions for which the optional {\tt undistLBound}
! and {\tt undistUBound} arguments must be specified.
!
! The optional {\tt distgridToArrayMap} argument provides the option to override
! the default DistGrid to Array dimension mapping. The entries of the
! {\tt distgridToArrayMap} array correspond to the DistGrid dimensions in
! sequence and assign a unique Array dimension to each DistGrid dimension.
! DistGrid and Array dimensions are indexed starting at {\tt 1} for the lowest
! dimension. A value of {\tt "0"} in the {\tt distgridToArrayMap} array 
! indicates that the respective DistGrid dimension is {\em not} mapped against
! any Array dimension. What this means is that the Array will be replicated 
! along this DistGrid dimension.
! 
! As a first example consider the case where a 1D Array
!EOE
!BOC
  call ESMF_ArraySpecSet(arrayspec, typekind=ESMF_TYPEKIND_R8, rank=1, rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! is created on the 2D DistGrid used during the previous section.
!EOE
!BOC
  array = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid, rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!  call ESMF_ArrayPrint(array, rc=rc)
!  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! Here the default DistGrid to Array dimension mapping is used which assigns
! the Array dimensions in sequence to the DistGrid dimensions starting with
! dimension "1". Extra DistGrid dimensions are considered replicator dimensions
! because the Array will be replicated along those dimensions. In the above
! example the 2nd DistGrid dimension will cause 1D Array pieces to be
! replicated along the DEs of the 2nd DistGrid dimension. Replication in the
! context of {\tt ESMF\_ArrayCreate()} does not mean that data values are
! communicated and replicated between different DEs, but it means that different
! DEs provide memory allocations for {\em identical} exclusive elements.
!
! Access to the data storage of an Array that has been replicated along 
! DistGrid dimensions is the same as for Arrays without replication.
!EOE
!BOC
  call ESMF_ArrayGet(array, localDeCount=localDeCount, rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOC
  allocate(larrayList(0:localDeCount-1))
  allocate(localDeToDeMap(0:localDeCount-1))
  call ESMF_ArrayGet(array, localarrayList=larrayList, &
    localDeToDeMap=localDeToDeMap, rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! The {\tt array} object was created without additional padding which means
! that the bounds of the Fortran array pointer correspond to the bounds of
! the exclusive region. The following loop will cycle through all local DEs, 
! print the DE number as well as the Fortran array pointer bounds. The bounds
! should be:
! \begin{verbatim}
!          lbound       ubound
!
! DE 0:      1            3         --+
! DE 2:      1            3         --|  1st replication set
! DE 4:      1            3         --+
!
! DE 1:      1            2         --+
! DE 3:      1            2         --|  2nd replication set
! DE 5:      1            2         --+
! \end{verbatim}
!EOE
!BOC
  do localDe=0, localDeCount-1
    call ESMF_LocalArrayGet(larrayList(localDe), myFarray1D, &
      datacopyflag=ESMF_DATACOPY_REFERENCE, rc=rc)
!EOC
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOC
    print *, "localPet: ", localPet, "DE ",localDeToDeMap(localDe)," [", &
      lbound(myFarray1D), ubound(myFarray1D),"]"
  enddo
  deallocate(larrayList)
  deallocate(localDeToDeMap)
  call ESMF_ArrayDestroy(array, rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! The Fortran array pointer in the above loop was of rank 1 because the
! Array object was of rank 1. However, the {\tt distgrid} object associated
! with {\tt array} is 2-dimensional! Consequently DistGrid based information
! queried from {\tt array} will be 2D. The {\tt distgridToArrayMap} and
! {\tt arrayToDistGridMap}
! arrays provide the necessary mapping to correctly associate DistGrid based 
! information with Array dimensions.
!
! The next example creates a 2D Array
!EOE
!BOC
  call ESMF_ArraySpecSet(arrayspec, typekind=ESMF_TYPEKIND_R8, rank=2, rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! on the previously used 2D DistGrid. By default, i.e. without the
! {\tt distgridToArrayMap}
! argument, both DistGrid dimensions would be associated with the two Array
! dimensions. However, the {\tt distgridToArrayMap} specified in the following
! call will only associate the second DistGrid dimension with the first Array 
! dimension. This will render the first DistGrid dimension a replicator
! dimension and the second Array dimension a tensor dimension for which 1D
! {\tt undistLBound} and {\tt undistUBound} arguments must be supplied.
!EOE
!BOC
  array = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid, &
    distgridToArrayMap=(/0,1/), undistLBound=(/11/), &
    undistUBound=(/14/), rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!  call ESMF_ArrayPrint(array, rc=rc)
!  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOC
  call ESMF_ArrayDestroy(array, rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! Finally, the same {\tt arrayspec} and {\tt distgrid} arguments are used to
! create a 2D Array that is fully replicated in both dimensions of the DistGrid.
! Both Array dimensions are now tensor dimensions and both DistGrid dimensions
! are replicator dimensions.
!EOE
!BOC
  array = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid, &
    distgridToArrayMap=(/0,0/), undistLBound=(/11,21/), &
    undistUBound=(/14,22/), rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! The result will be an Array with local lower bound (/11,21/) and upper bound
! (/14,22/) on all 6 DEs of the DistGrid.
!BOC
  call ESMF_ArrayDestroy(array, rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOC
  call ESMF_DistGridDestroy(distgrid, rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!BOE
! Replicated Arrays can also be created from existing local Fortran arrays.
! The following Fortran array allocation will provide a 3 x 10 array on each
! PET. 
!EOE
!BOC
  allocate(myFarray2D(3,10))
!EOC
!BOE
! Assuming a petCount of 4 the following DistGrid defines a 2D index space
! that is distributed across the PETs along the first dimension.
!EOE
!BOC
  distgrid = ESMF_DistGridCreate(minIndex=(/1,1/), maxIndex=(/40,10/), rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! The following call creates an Array object on the above distgrid using
! the locally existing {\tt myFarray2D} Fortran arrays. The difference 
! compared to the case with automatic memory allocation is that instead of
! {\tt arrayspec} the Fortran array is provided as argument. Furthermore,
! the {\tt undistLBound} and {\tt undistUBound} arguments can be omitted,
! defaulting into Array tensor dimension lower bound of 1 and an upper
! bound equal to the size of the respective Fortran array dimension.
!EOE
!BOC
  array = ESMF_ArrayCreate(farray=myFarray2D, distgrid=distgrid, &
    indexflag=ESMF_INDEX_DELOCAL, distgridToArrayMap=(/0,2/), rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! The {\tt array} object associates the 2nd DistGrid dimension with the 2nd
! Array dimension. The first DistGrid dimension is not associated with any
! Array dimension and will lead to replication of the Array along the DEs of
! this direction.
!EOE
!BOC
  call ESMF_ArrayDestroy(array, rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOC
  call ESMF_DistGridDestroy(distgrid, rc=rc)
!EOC
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
  deallocate(myFarray2D)

!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
!!!! UNTIL FURTHER IMPLEMENTATION SKIP SECTIONS OF THIS EXAMPLE >>>>>>>>>>>>>>>>
#ifdef NOSKIP   
!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>


!BOEI
!
! \subsubsection{Array for a bipolar DistGrid}
!
! The Murray tripolar grid possesses two poles in the arctic region. The 
! implications of the bipolar grid of the northern hemisphere on the DistGrid
! and Array objects are discussed in this section.
!
! The bipolar region can either be defined as a single tile DistGrid or
! it can be composed of two or more separate tiles. In this example two
! tiles, each covering half of the index space of the bipolar region, will be
! used.
!
! The index space of the bipolar region remains logically rectangular (LR) and
! is assumed to be of size 360 x 50 elements for this example. The index order
! in the example is assumed $i,j$. The line for $j=1$ corresponds to a line of 
! constant latitude in the spherical coordinate system and the {\em bipolar
! fold} is along $j=50$. Two equal sized tiles of each 180 x 50 elements need
! to be connected in a way that corresponds to the bipolar topology.
! 
!EOEI
!BOCI
  allocate(connectionList(2*2+2,1))   ! 1 connection: the bipolar fold
  call ESMF_DistGridConnection(connection=connectionList(:,1), &
    tileIndexA=1, tileIndexB=2, &
    positionVector=(/179, 99/), orientationVector=(/-1, -2/), rc=rc)
!EOCI  
!BOEI
! With this {\tt connectionList} it is now
! possible to define a DistGrid object that captures the index space topology
! for a bipolar grid. The DistGrid consists of two tiles which need to be
! provided in {\tt minIndex} and {\tt maxIndex} list arguments.
!EOEI
!BOCI
  allocate(minIndex(2,2), maxIndex(2,2), regDecomp(2,2))
  minIndex(:,1) = (/1,1/)              ! first tile
  maxIndex(:,1) = (/180,50/)           ! first tile
  regDecomp(:,1) = (/petCount/2, 1/)    ! first tile
  minIndex(:,2) = (/1,1/)              ! second tile
  maxIndex(:,2) = (/180,50/)           ! second tile
  regDecomp(:,2) = (/petCount/2, 1/)    ! second tile
  
  distgrid = ESMF_DistGridCreate(minIndex=minIndex, maxIndex=maxIndex, &
    regDecomp=regDecomp, connectionList=connectionList, rc=rc)
!EOCI  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOEI
! The decomposition described by {\tt regDecomp} assumes that there is an even 
! number of PETs available in the current Component. The decomposition will be 
! into as many DEs as PETs. Half of the DEs handle the first tile and the other
! half of DEs handle the second tile.
!
! In order to create a 2D Array on this DistGrid for single precision
! real data the ArraySpec variable must be set correctly.
!EOEI
!BOCI
  call ESMF_ArraySpecSet(arrayspec, typekind=ESMF_TYPEKIND_R4, rank=2, rc=rc)
!EOCI
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOEI
! Finally the Array objects can be created for this {\tt arrayspec} and {\tt 
! distgrid}. A scalar tracer Array will be created without halo padding.
!EOEI
!BOCI
  arrayTracer = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid, rc=rc)
!EOCI
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOEI
! Next an Array is created for a scalar with space for a halo width of one 
! element in each direction.
!EOEI
!BOCI
  arrayScalar = ESMF_ArrayCreate(arrayspec=arrayspec, distgrid=distgrid, &
    totalLWidth=(/1,1/), totalUWidth=(/1,1/), rc=rc)
!EOCI
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

  call ESMF_ArrayDestroy(arrayTracer, rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

  call ESMF_ArrayDestroy(arrayScalar, rc=rc)
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

  call ESMF_DistGridDestroy(distgrid, rc=rc)

!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#endif
!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

!===============================================================================

!BOE
! \subsubsection{Shared memory features: DE pinning, sharing, and migration}
! \label{Array_shared_memory_features}
!
! Practically all modern computer systems today utilize multi-core processors,
! supporting the concurrent execution of multiple hardware threads.
! A number of these multi-core processors are commonly packaged into the same
! compute node, having access to the same physical memory. Under ESMF each
! hardware thread (or core) is identified as a unique Processing Element (PE).
! The collection of PEs that share the same physical memory (i.e. compute node)
! is referred to as a Single System Image (SSI). The ESMF Array class implements
! features that allow the user to leverage the shared memory within each SSI to
! efficiently exchange data without copies or explicit communication calls.
!
! The software threads executing an ESMF application on the hardware, and that
! ESMF is aware of, are referred to as Persistent Execution Threads (PETs). In
! practice a PET can typically be thought of as an MPI rank, i.e. an OS process,
! defining its own private virtual address space (VAS). 
! The ESMF Virtual Machine (VM) class keeps track of the mapping between PETs
! and PEs, and their location on the available SSIs.
!
! When an ESMF Array object is created, the specified DistGrid indicates how
! many Decomposition Elements (DEs) are created. Each DE has its own memory
! allocation to hold user data. The DELayout, referenced by the DistGrid,
! determines which PET is considered the {\em owner} of each of the DEs. Queried
! for the local DEs, the Array object returns the list of DEs that are owned by
! the local PET making the query.
!
! By default DEs are {\em pinned} to the PETs under which they were created.
! The memory allocation associated with a specific DE is only defined in the
! VAS of the PET to which the DE is pinned. As a consequence, only the PET
! owning a DE has access to its memory allocation.
!
! On shared memory systems, however, ESMF allows DEs to be pinned to SSIs
! instead of PETs. In this case the PET under which a DE was created is still
! consider the owner, but now {\em all} PETs under the same SSI have access to
! the DE. For this the memory allocation associated with the DE is mapped into
! the VAS of all the PETs under the SSI.
!
! To create an Array with each DE pinned to SSI instead of PET, first query the
! VM for the available level of support.
!EOE

!BOC
  call ESMF_VMGet(vm, ssiSharedMemoryEnabledFlag=ssiSharedMemoryEnabled, rc=rc)
!EOC  
  if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOC
  if (ssiSharedMemoryEnabled) then
!EOC  
!BOE
! Knowing that the SSI shared memory feature is available, it is now possible
! to create an Array object with DE to SSI pinning.
!EOE

!BOC
    distgrid = ESMF_DistGridCreate(minIndex=(/1,1/), maxIndex=(/40,10/), rc=rc)
!EOC
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOC
    array = ESMF_ArrayCreate(typekind=ESMF_TYPEKIND_R8, distgrid=distgrid, &
      pinflag=ESMF_PIN_DE_TO_SSI, rc=rc)
!EOC  
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!BOE
! Just as in the cases discussed before, where the same DistGrid was used, a
! default DELayout with as many DEs as PETs in the VM is constructed. Setting
! the {\tt pinflag} to {\tt ESMF\_PIN\_DE\_TO\_SSI} does not change the
! fact that each PET owns exactly one of the DEs. However, assuming that this
! code is run on a set of PETs that are all located under the same SSI, every
! PET now has {\em access} to all of the DEs. The situation can be observed by
! querying for both the {\tt localDeCount}, and the {\tt ssiLocalDeCount}.
!EOE
!BOC
    call ESMF_ArrayGet(array, localDeCount=localDeCount, &
      ssiLocalDeCount=ssiLocalDeCount, rc=rc)
!EOC  
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!BOE
! \begin{sloppypar}
! Assuming execution on 4 PETs, all located on the same SSI, the values of the
! returned variable are {\tt localDeCount==1} and {\tt ssiLocalDeCount==4} on 
! all of the PETs. The mapping between each PET's local DE, and the global DE
! index is provided through the {\tt localDeToDeMap} array argument. The amount
! of mapping information returned is dependent on how large {\tt localDeToDeMap}
! has been sized by the user. For {\tt size(localDeToDeMap)==localDeCount},
! only mapping information for those DEs {\em owned} by the local PET is filled
! in. However for {\tt size(localDeToDeMap)==ssiLocalDeCount}, mapping
! information for all locally {\em accessible} DEs is returned, including
! those owned by other PETs on the same SSI.
! \end{sloppypar}
!EOE
!BOC
    allocate(localDeToDeMap(0:ssiLocalDeCount-1))
    call ESMF_ArrayGet(array, localDeToDeMap=localDeToDeMap, rc=rc)
!EOC  
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!BOE
! The first {\tt localDeCount} entries of {\tt localDeToDeMap} are always the
! global DE indices of the DEs {\em owned} by the local PET. The remaining 
! {\tt ssiLocalDeCount}-{\tt localDeCount} entries are the global DE indices of
! DEs {\em shared} by other PETs. The ordering of the shared DEs is from
! smallest to greatest, excluding the locally owned DEs, which were already
! listed at the beginning of {\tt localDeToDeMap}. For the current case, again
! assuming execution on 4 PETs all located on the same SSI, we expect the
! following situation:
!
! PET 0: {\tt localDeToDeMap}=={\tt(/0,1,2,3/)}\newline
! PET 1: {\tt localDeToDeMap}=={\tt(/1,0,2,3/)}\newline
! PET 2: {\tt localDeToDeMap}=={\tt(/2,0,1,3/)}\newline
! PET 3: {\tt localDeToDeMap}=={\tt(/3,0,1,2/)}\newline
!
! Each PET can access the memory allocations associated with {\em all} of the
! DEs listed in the {\tt localDeToDeMap} returned by the Array object. Direct
! access to the Fortran array pointer of a specific memory allocation is
! available through {\tt ESMF\_ArrayGet()}. Here each PET queries for the
! {\tt farrayPtr} of {\tt localDe==2}, i.e. the 2nd shared DE.
!EOE
!BOC
    call ESMF_ArrayGet(array, farrayPtr=myFarray, localDe=2, rc=rc)
!EOC  
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!BOE
! Now variable {\tt myFarray} on PETs 0 and 1 both point to the {\em same}
! memory allocation for global DE 2. Both PETs have access to the same
! piece of shared memory! The same is true for PETs 2 and 3, pointing to the
! shared memory allocation of global DE 1.

! It is important to note that all of the typical considerations surrounding
! shared memory programming apply when accessing shared DEs! Proper
! synchronization between PETs accessing shared DEs is critical to avoid
! {\em race conditions}. Also performance issues like {\em false sharing}
! need to be considered for optimal use.
!
! For a simple demonstration, PETs 0 and 2 fill the entire memory allocation of
! DE 2 and 1, respectively, to a unique value.
!EOE
!BOC
    if (localPet==0) then
      myFarray = 12345.6789d0
    else if (localPet==2) then
      myFarray = 6789.12345d0
    endif
!EOC
!BOE
! Here synchronization is needed before any PETs that share access to the same
! DEs can safely access the data without race condition. The Array class provides
! a simple synchronization method that can be used.
!EOE
!BOC
    call ESMF_ArraySync(array, rc=rc) ! prevent race condition
!EOC  
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! Now it is safe for PETs 1 and 3 to access the shared DEs. We expect to find
! the data that was set above. For simplicity of the code only the first
! array element is inspected here.
!EOE
!BOC
    if (localPet==1) then
      if (abs(myFarray(1,1)-12345.6789d0)>1.d10) print *, "bad data detected"
    else if (localPet==3) then
      if (abs(myFarray(1,1)-6789.12345d0)>1.d10) print *, "bad data detected"
    endif
!EOC

!BOE
! Working with shared DEs requires additional bookkeeping on the user code
! level. In some situations, however, DE sharing is simply used as a mechanism
! to {\em move} DEs between PETs without requiring data copies. One practical
! application of this case is the transfer of an Array between two components,
! both of which use the same PEs, but run with different number of PETs.
! These would typically be sequential components that use OpenMP on the user
! level with varying threading levels.
!
! DEs that are pinned to SSI can be moved or {\em migrated} to any PET within
! the SSI. This is accomplished by creating a new Array object from an
! existing Array that was created with {\tt pinflag=ESMF\_PIN\_DE\_TO\_SSI}.
! The information of how the DEs are to migrate between the old and the new
! Array is provided through a DELayout object. This object must have the
! same number of DEs and describes how they map to the PETs on the current VM.
! If this is in the context of a different component, the number of PETs might
! differ from the original VM under which the existing Array was created. This
! situation is explicitly supported, still the number of DEs must match.
!
! Here a simple DELayout is created on the same 4 PETs, but with rotated
! DE ownerships:
!
! DE 0 -> PET 1 (old PET 0)\newline
! DE 1 -> PET 2 (old PET 1)\newline
! DE 2 -> PET 3 (old PET 2)\newline
! DE 3 -> PET 0 (old PET 3)\newline
!EOE
!BOC
    delayout = ESMF_DELayoutCreate(petMap=(/1,2,3,0/), rc=rc) ! DE->PET mapping
!EOC  
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!BOE
! The creation of the new Array is done by reference, i.e. 
! {\tt datacopyflag=ESMF\_DATACOPY\_REFERENCE}, since the new Array does
! not create its own memory allocations. Instead the new Array references the
! shared memory resources held by the incoming Array object.
!EOE
!BOC
    arrayMigrated = ESMF_ArrayCreate(array, delayout=delayout, &
      datacopyflag=ESMF_DATACOPY_REFERENCE, rc=rc)
!EOC  
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!BOE
! Querying {\tt arrayMigrated} for the number of local DEs will return 1 on
! each PET. Sizing the {\tt localDeToDeMap} accordingly and querying for it.
!EOE
!BOC
    deallocate(localDeToDeMap) ! free previous allocation
    allocate(localDeToDeMap(0:1))
    call ESMF_ArrayGet(arrayMigrated, localDeToDeMap=localDeToDeMap, rc=rc)
!EOC  
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!BOE
! This yields the following expected outcome:
!
! PET 0: {\tt localDeToDeMap}=={\tt(/1/)}\newline
! PET 1: {\tt localDeToDeMap}=={\tt(/2/)}\newline
! PET 2: {\tt localDeToDeMap}=={\tt(/3/)}\newline
! PET 3: {\tt localDeToDeMap}=={\tt(/0/)}\newline
!
! On each PET the respective Fortran array pointer is returned by the Array.
!EOE
!BOC
    call ESMF_ArrayGet(arrayMigrated, farrayPtr=myFarray, rc=rc)
!EOC  
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)

!BOE
! The same situation could have been achieved with the original {\tt array}.
! However, it would have required first finding the correct local DE
! for the target global DE on each PET, and then querying {\tt array}
! accordingly. If needed more repeatedly, this bookkeeping would need to be
! kept in a user code data structure. The DE migration feature on the other
! hand provides a formal way to create a standard ESMF Array object that can be
! used directly in any Array level method as usual, letting ESMF handle the
! extra bookkeeping needed.
!EOE
    deallocate(localDeToDeMap)

!BOE
! Before destroying an Array whose DEs are shared between PETs, it is
! advisable to issue one more synchronization. This prevents cases where a
! PET still might be accessing a shared DE, while the owner PET is already
! destroying the Array, therefore deallocating the shared memory resource.
!EOE
!BOC
    call ESMF_ArraySync(array, rc=rc) ! prevent race condition
!EOC
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOC
    call ESMF_ArrayDestroy(array, rc=rc)
!EOC
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOE
! Remember that {\tt arrayMigrated} shares the same memory allocations that were
! held by {\tt array}. Array {\tt arrayMigrated} must therefore not be used
! beyond the life time of {\tt array}. Best to destroy it now.
!EOE
!BOC
    call ESMF_ArrayDestroy(arrayMigrated, rc=rc)
!EOC
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
    call ESMF_DistGridDestroy(distgrid, rc=rc)
    if (rc /= ESMF_SUCCESS) call ESMF_Finalize(endflag=ESMF_END_ABORT)
!BOC
  endif ! ending the ssiSharedMemoryEnabled conditional
!EOC

!===============================================================================

10 continue
  ! IMPORTANT: ESMF_STest() prints the PASS string and the # of processors in the log
  ! file that the scripts grep for.
  call ESMF_STest((finalrc.eq.ESMF_SUCCESS), testname, failMsg, result, ESMF_SRCLINE)

  
  if (rc/=ESMF_SUCCESS) finalrc = ESMF_FAILURE
  if (finalrc==ESMF_SUCCESS) then
    print *, "PASS: ESMF_ArrayEx.F90"
  else
    print *, "FAIL: ESMF_ArrayEx.F90"
  endif

  call ESMF_Finalize(rc=rc)
  
end program
