#include "cppdefs.h"
#ifdef MPI

# if (!defined MP_3PTS) && (!defined MP_1PTS)
      subroutine MessPass2D_tile (Istr,Iend,Jstr,Jend, A)
# elif defined MP_1PTS
      subroutine MessPass2D_1pts_tile (Istr,Iend,Jstr,Jend, A)
# else
      subroutine MessPass2D_3pts_tile (Istr,Iend,Jstr,Jend, A)
# endif
!
!======================================================================
!
! This subroutine is designed for CROCO-MPI code. It exchanges domain
! boundary information, including 2 (or 3) ghost-cells in each direction.
!
! Ping Wang 9/15/99.
! Patrick Marchesiello 2012: generic number of ghost-cells Npts
!
!======================================================================
!     implicit none
# include "param.h"
# include "scalars.h"
# include "mpi_cpl.h"
      include 'mpif.h'
      integer Npts
# if (!defined MP_3PTS) && (!defined MP_1PTS)
      parameter (Npts=2)
# elif defined MP_1PTS
      parameter (Npts=1)
# else
      parameter (Npts=3)
# endif
      logical, parameter :: use_host_device = .TRUE.
# include "mpi_buffers.h"
!
! Nb of boundary points involved in communication
!
      integer ipts,jpts 
      real A(GLOBAL_2D_ARRAY)
CSDISTRIBUTE_RESHAPE A(BLOCK_PATTERN) BLOCK_CLAUSE
      integer Istr,Iend,Jstr,Jend, i,j, isize,jsize,ksize,  iter,
     &        req(8), status(MPI_STATUS_SIZE,8), ierr, mdii,mdjj
     
# ifdef  BAND_DEBUG         
      real,dimension(Npts)        :: checksum
# endif
!
# include "compute_message_bounds.h"

# ifdef AGRIF
         sub_X=Lm
         size_X=MAXNPTS*(sub_X+2*MAXNPTS)-1  !7+Npts*sub_X
         sub_E=Mm
         size_E=MAXNPTS*(sub_E+2*MAXNPTS)-1  !7+Npts*sub_E
         if (.not. allocated(ibuf_sndN)) then
         Allocate(ibuf_sndN(0:size_X), ibuf_revN(0:size_X), 
     &     ibuf_sndS(0:size_X), ibuf_revS(0:size_X),
     &     jbuf_sndW(0:size_E), jbuf_sndE(0:size_E),
     &     jbuf_revW(0:size_E), jbuf_revE(0:size_E))

         endif
# endif
!
      ksize=Npts*Npts
      isize=Npts*ishft                  ! sizes for side messages
      jsize=Npts*jshft                  ! in XI and ETA directions

# define write !
c*      write(*,'(2(6x,A3,I2,2x,A5,I3,2x,A5,I3))')
c*     &        'ii=',ii,'imin=',imin,'imax=',imax,
c*     &        'jj=',jj,'jmin=',jmin,'jmax=',jmax
                            !  Message passing split into two stages
                            !  in order to optimize Send-Recv pairing
                            !  in such a way that if one subdomain
      do iter=0,1           !  sends message to, say, its WESTERN
        mdii=mod(ii+iter,2) !  neighbor, that neighbor is preparing
        mdjj=mod(jj+iter,2) !  to receive this message first (i.e.
                            !  message coming from its EASTERN side),
                            !  rather than send his WEST
                            !  bound message, similarly to the first
                            !  subdomain.
!
! Prepare to receive and send: sides....
        if (mdii.eq.0) then
          if (WEST_INTER2) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
            do j=jmin,jmax
              do ipts=1,Npts
                jbuf_sndW(j-jmin+(ipts-1)*jshft)=A(ipts,j)
              enddo
            enddo
!$acc update host(jbuf_sndW) if(.not.use_host_device)
!$acc host_data if(compute_on_device.and.use_host_device) 
!$acc& use_device(jbuf_revW, jbuf_sndW)
            call MPI_Irecv (jbuf_revW, jsize, MPI_DOUBLE_PRECISION,
     &                         p_W, 2, MPI_COMM_WORLD, req(1), ierr)
            call MPI_Send  (jbuf_sndW, jsize, MPI_DOUBLE_PRECISION,
     &                         p_W, 1, MPI_COMM_WORLD,         ierr)
!$acc end host_data
          endif
        else
          if (EAST_INTER2) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
            do j=jmin,jmax
              do ipts=1,Npts
                jbuf_sndE(j-jmin+(ipts-1)*jshft)=A(Lmmpi-Npts+ipts,j)
              enddo
            enddo
!$acc host_data if(compute_on_device) use_device(jbuf_revE, jbuf_sndE)
            call MPI_Irecv (jbuf_revE, jsize, MPI_DOUBLE_PRECISION,
     &                         p_E, 1, MPI_COMM_WORLD, req(2), ierr)
            call MPI_Send  (jbuf_sndE, jsize, MPI_DOUBLE_PRECISION,
     &                         p_E, 2, MPI_COMM_WORLD,         ierr)
!$acc end host_data
          endif
        endif

        if (mdjj.eq.0) then
          if (SOUTH_INTER2) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
            do jpts=1,Npts
              do i=imin,imax
                ibuf_sndS(i-imin+(jpts-1)*ishft)=A(i,jpts)
              enddo
            enddo
!$acc update host(ibuf_sndS) if(.not.use_host_device)
!$acc host_data if(compute_on_device.and.use_host_device)
!$acc&  use_device(ibuf_revS, ibuf_sndS)
            call MPI_Irecv (ibuf_revS, isize, MPI_DOUBLE_PRECISION,
     &                         p_S, 4, MPI_COMM_WORLD, req(3), ierr)
            call MPI_Send  (ibuf_sndS, isize, MPI_DOUBLE_PRECISION,
     &                         p_S, 3, MPI_COMM_WORLD,         ierr)
!$acc end host_data
          endif
        else
          if (NORTH_INTER2) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
            do jpts=1,Npts
              do i=imin,imax
                ibuf_sndN(i-imin+(jpts-1)*ishft)=A(i,Mmmpi-Npts+jpts)
              enddo
            enddo
!$acc update host(ibuf_sndN) if(.not.use_host_device)
!$acc host_data if(compute_on_device.and.use_host_device)
!$acc&  use_device(ibuf_revN, ibuf_sndN)
            call MPI_Irecv (ibuf_revN, isize, MPI_DOUBLE_PRECISION,
     &                         p_N, 3, MPI_COMM_WORLD, req(4), ierr)
            call MPI_Send  (ibuf_sndN, isize, MPI_DOUBLE_PRECISION,
     &                         p_N, 4, MPI_COMM_WORLD,         ierr)
!$acc end host_data
          endif
        endif
!
! ...corners:
!

        if (mdii.eq.0) then
          if (CORNER_SW) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
            do jpts=1,Npts
              do ipts=1,Npts
                buf_snd1(ipts+Npts*(jpts-1))=A(ipts,jpts)
              enddo
            enddo
!$acc host_data if(compute_on_device) use_device(buf_rev1,buf_snd1)
            call MPI_Irecv (buf_rev1,ksize, MPI_DOUBLE_PRECISION,  p_SW,
     &                               6, MPI_COMM_WORLD, req(5),ierr)
            call MPI_Send  (buf_snd1,ksize, MPI_DOUBLE_PRECISION,  p_SW,
     &                               5, MPI_COMM_WORLD,        ierr)
!$acc end host_data
          endif
        else
          if (CORNER_NE) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
            do jpts=1,Npts
              do ipts=1,Npts
                buf_snd2(ipts+Npts*(jpts-1))=
     &                                A(Lmmpi-Npts+ipts,Mmmpi-Npts+jpts)
              enddo
            enddo
!$acc host_data if(compute_on_device) use_device(buf_rev2,buf_snd2)
            call MPI_Irecv (buf_rev2,ksize, MPI_DOUBLE_PRECISION,  p_NE,
     &                               5, MPI_COMM_WORLD, req(6),ierr)
            call MPI_Send  (buf_snd2,ksize, MPI_DOUBLE_PRECISION,  p_NE,
     &                               6, MPI_COMM_WORLD,        ierr)
!$acc end host_data
          endif
        endif
        if (mdii.eq.1) then
          if (CORNER_SE) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
            do jpts=1,Npts
              do ipts=1,Npts
                buf_snd3(ipts+Npts*(jpts-1))=
     &                                A(Lmmpi-Npts+ipts,jpts)
              enddo
            enddo
!$acc host_data if(compute_on_device) use_device(buf_rev3,buf_snd3)
            call MPI_Irecv (buf_rev3,ksize, MPI_DOUBLE_PRECISION,  p_SE,
     &                               8, MPI_COMM_WORLD, req(7), ierr)
            call MPI_Send  (buf_snd3,ksize, MPI_DOUBLE_PRECISION,  p_SE,
     &                               7, MPI_COMM_WORLD,         ierr)
!$acc end host_data
          endif
        else
          if (CORNER_NW) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
            do jpts=1,Npts
              do ipts=1,Npts
                buf_snd4(ipts+Npts*(jpts-1))=
     &                                A(ipts,Mmmpi-Npts+jpts)
              enddo
            enddo
!$acc host_data if(compute_on_device) use_device(buf_rev4,buf_snd4)
            call MPI_Irecv (buf_rev4, ksize, MPI_DOUBLE_PRECISION, p_NW,
     &                               7, MPI_COMM_WORLD, req(8), ierr)
            call MPI_Send  (buf_snd4, ksize, MPI_DOUBLE_PRECISION, p_NW,
     &                               8, MPI_COMM_WORLD,         ierr)
!$acc end host_data
          endif
        endif
      enddo   !<-- iter
!
! Wait for completion of receive and fill ghost points: sides...
!
      if (WEST_INTER2) then
        call MPI_Wait (req(1),status(1,1),ierr)
!$acc update device(jbuf_revW) if(.not.use_host_device)
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
# ifdef  BAND_DEBUG         
          checksum(:) = 0.
# endif          
c           print *,"chkbandname:",chkbandname,"---"
        do j=jmin,jmax
          do ipts=1,Npts
# ifdef  BAND_DEBUG         
c            if ( chkbandname .ne. 'none' ) then
            if ( A(ipts-Npts,j)==jbuf_revW(j-jmin+(ipts-1)*jshft) ) 
     &           checksum(ipts) = checksum(ipts)+1
c            endif
# endif
           A(ipts-Npts,j)=jbuf_revW(j-jmin+(ipts-1)*jshft)
          enddo
        enddo
        ! 
# ifdef  BAND_DEBUG         
c         if ( chkbandname .ne. 'none') then
          do ipts=1,Npts
c             print *,"checksum(ipts)=",checksum(ipts)
c             print *,"jnin,jmax=",jmin,jmax,":",ipts
            if( nint(checksum(ipts)) == (jmax-jmin+1) ) then
               print *,mynode,"ipts=",int(ipts,1),"WEST_INTER2 "//
     &          trim(chkbandname)//': ghostcell '
c                stop
            endif
          enddo
          checksum(:) = 0.
c          endif
# endif        
      endif
      if ( WEST_INTER .and. .not. WEST_INTER2 ) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
        do j=jmin,jmax
          do ipts=1,Npts
           A(ipts-Npts,j)=A(ipts,j)
          enddo
        enddo
      endif

      if (EAST_INTER2) then
        call MPI_Wait (req(2),status(1,2),ierr)
!$acc update device(jbuf_revE) if(.not.use_host_device)
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
# ifdef  BAND_DEBUG         
        checksum(:) = 0.
# endif        
        do j=jmin,jmax
          do ipts=1,Npts
# ifdef  BAND_DEBUG         
c            if ( chkbandname .ne. 'none' ) then
            if (  A(Lmmpi+ipts,j)==jbuf_revE(j-jmin+(ipts-1)*jshft) ) 
     &          checksum(ipts) = checksum(ipts)+1
c            endif
# endif
           A(Lmmpi+ipts,j)=jbuf_revE(j-jmin+(ipts-1)*jshft)
          enddo
        enddo
# ifdef  BAND_DEBUG         
c         if ( chkbandname .ne. 'none') then
          do ipts=1,Npts
c             print *,"checksum(ipts)=",checksum(ipts)
c             print *,"jnin,jmax=",jmin,jmax,":",ipts
            if( nint(checksum(ipts)) == (jmax-jmin+1) ) then
               print *,mynode,"ipts=",int(ipts,1),"EAST_INTER2 "//
     &          trim(chkbandname)//': ghostcell '
c                stop
            endif
          enddo
          checksum(:) = 0.
c          endif
# endif        
      endif
      if ( EAST_INTER .and. .not. EAST_INTER2 ) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
        do j=jmin,jmax
          do ipts=1,Npts
           A(Lmmpi+ipts,j)=A(Lmmpi-Npts+ipts,j)
          enddo
        enddo
      endif

      if (SOUTH_INTER2) then
        call MPI_Wait (req(3),status(1,3),ierr)
!$acc update device(ibuf_revS) if(.not.use_host_device)
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
# ifdef  BAND_DEBUG         
           checksum(:) = 0.
# endif           
       do jpts=1,Npts
          do i=imin,imax
# ifdef  BAND_DEBUG         
c            if ( chkbandname .ne. 'none' ) then
            if ( A(i,jpts-Npts)==ibuf_revS(i-imin+(jpts-1)*ishft) ) 
     &          checksum(jpts) = checksum(jpts)+1
c            endif
# endif
           A(i,jpts-Npts)=ibuf_revS(i-imin+(jpts-1)*ishft)
          enddo
        enddo
# ifdef  BAND_DEBUG         
c         if ( chkbandname .ne. 'none') then
          do jpts=1,Npts
c             print *,"checksum(ipts)=",checksum(ipts)
c             print *,"jnin,jmax=",jmin,jmax,":",ipts
            if( nint(checksum(jpts)) == (imax-imin+1) ) then
               print *,mynode,"jpts=",int(jpts,1),"SOUTH_INTER2 "//
     &          trim(chkbandname)//': ghostcell '
c                stop
            endif
          enddo
          checksum(:) = 0.
c          endif
# endif        
      endif
      if ( SOUTH_INTER .and. .not. SOUTH_INTER2 ) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
        do jpts=1,Npts
          do i=imin,imax
           A(i,jpts-Npts)=A(i,jpts)
          enddo
        enddo
      endif

      if (NORTH_INTER2) then
        call MPI_Wait (req(4),status(1,4),ierr)
!$acc update device(ibuf_revN) if(.not.use_host_device)
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
# ifdef  BAND_DEBUG         
        checksum(:) = 0.
# endif        
        do jpts=1,Npts
          do i=imin,imax
# ifdef  BAND_DEBUG         
c            if ( chkbandname .ne. 'none' ) then
            if ( A(i,Mmmpi+jpts)==ibuf_revN(i-imin+(jpts-1)*ishft) ) 
     &          checksum(jpts) = checksum(jpts)+1
c            endif
# endif
            A(i,Mmmpi+jpts)=ibuf_revN(i-imin+(jpts-1)*ishft)
          enddo
        enddo
# ifdef  BAND_DEBUG         
c         if ( chkbandname .ne. 'none') then
          do jpts=1,Npts
c             print *,"checksum(ipts)=",checksum(ipts)
c             print *,"jnin,jmax=",jmin,jmax,":",ipts
            if( nint(checksum(jpts)) == (imax-imin+1) ) then
               print *,mynode,"jpts=",int(jpts,1),"NORTH_INTER2 "//
     &          trim(chkbandname)//': ghostcell '
c                stop
            endif
          enddo
          checksum(:) = 0.
c          endif
# endif        
      endif
      if (NORTH_INTER .AND. .not. NORTH_INTER2) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
        do jpts=1,Npts
          do i=imin,imax
            A(i,Mmmpi+jpts)=A(i,Mmmpi-Npts+jpts)
          enddo
        enddo
      endif

!
! ...corners:
!
      if ( CORNER_SW) then
        call MPI_Wait (req(5),status(1,5),ierr)
!$acc update device(buf_rev1) if(.not.use_host_device)
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
        do jpts=1,Npts
          do ipts=1,Npts
          A(ipts-Npts,jpts-Npts)=buf_rev1(ipts+Npts*(jpts-1))
          enddo
        enddo
      endif
      if (.not.CORNER_SW .and.
     &  SOUTH_INTER .and.WEST_INTER ) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
        do jpts=1,Npts
          do ipts=1,Npts
           A(ipts-Npts,jpts-Npts)=A(ipts,jpts)
          enddo
        enddo
       endif

      if ( CORNER_NE) then
        call MPI_Wait (req(6),status(1,6),ierr)
!$acc update device(buf_rev2) if(.not.use_host_device)
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
        do jpts=1,Npts
          do ipts=1,Npts
            A(Lmmpi+ipts,Mmmpi+jpts)=buf_rev2(ipts+Npts*(jpts-1))
          enddo
        enddo
      endif
      if (.not.CORNER_NE .and.
     &  NORTH_INTER .and.EAST_INTER ) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
        do jpts=1,Npts
          do ipts=1,Npts
            A(Lmmpi+ipts,Mmmpi+jpts)=A(Lmmpi+ipts-NPts,Mmmpi+jpts-Npts)
          enddo
        enddo
       endif

      if (CORNER_SE) then
        call MPI_Wait (req(7),status(1,7),ierr)
!$acc update device(buf_rev3) if(.not.use_host_device)
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
        do jpts=1,Npts
          do ipts=1,Npts
            A(Lmmpi+ipts,jpts-Npts)=buf_rev3(ipts+Npts*(jpts-1))
          enddo
        enddo
      endif
      if (.not. CORNER_SE .and.
     &  SOUTH_INTER .and. EAST_INTER ) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
        do jpts=1,Npts
          do ipts=1,Npts
            A(Lmmpi+ipts,jpts-Npts)=A(Lmmpi+ipts-Npts,jpts)
          enddo
        enddo
      endif


      if (CORNER_NW) then
        call MPI_Wait (req(8),status(1,8),ierr)
!$acc update device(buf_rev4) if(.not.use_host_device)
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
        do jpts=1,Npts
          do ipts=1,Npts
            A(ipts-Npts,Mmmpi+jpts)=buf_rev4(ipts+Npts*(jpts-1))
          enddo
        enddo
      endif
      if (.not. CORNER_NW .and.
     &   NORTH_INTER .and.  WEST_INTER ) then
!$acc parallel loop if(compute_on_device) independent
!$acc&  default(present) collapse(2)
       do jpts=1,Npts
          do ipts=1,Npts
            A(ipts-Npts,Mmmpi+jpts)=A(ipts,Mmmpi+jpts-NPts)
          enddo
        enddo
       endif


c      write(*,*) 'MessPass2D:    ', mynode,' exit'


      return
      end

# if (!defined MP_3PTS) && (!defined MP_1PTS)
#  define MP_3PTS
#  include "MessPass2D.F"
#  undef MP_3PTS
#  ifndef MP_1PTS
#   define MP_1PTS
#   include "MessPass2D.F"
#   undef MP_1PTS
#  endif
# endif
#else
      subroutine MessPass2D_empty
      return
      end
#endif
