158 ,ig_cpu_neighbor_info&
159 ,ig_nneighbor_all_kind&
161 ,ig_mpi_buffer_offset&
165 ,ig_mpi_request_send&
166 ,ig_mpi_request_recv&
172 ,lg_output_debug_file&
174 ,ig_mpi_buffer_sizemax
180 integer,
parameter :: nface = 6
181 integer,
parameter :: nedge = 12
182 integer,
parameter :: nnode = 8
184 real ,
dimension(:,:),
allocatable :: gll_coord_send
185 real ,
dimension(:,:),
allocatable :: gll_coord_recv
187 integer,
dimension(MPI_STATUS_SIZE) :: statut
191 integer :: myunit_debug
192 integer :: icpu_neighbor
195 integer :: ngll_duplicate
201 integer :: ngll_unique
202 integer :: mpi_buffer_sizemax
203 integer :: buffer_sizemax
204 integer,
dimension(:),
allocatable :: buffer_gll_duplicate
206 character(len= 6) :: cl_rank
207 character(len=255) :: info
209 if (ig_myrank == 0)
then
210 write(ig_lst_unit,
'(" ",/,a)')
"creating mpi buffers between cpu myrank and its neighbors..."
211 call flush(ig_lst_unit)
214 buffer_sizemax = (6*ig_ngll*ig_ngll + 12*ig_ngll + 8) * ig_nhexa_outer
216 ios =
init_array_int(buffer_gll_duplicate,buffer_sizemax,
"buffer_gll_duplicate")
218 if (lg_async_mpi_comm)
then
220 ios =
init_array_int(ig_mpi_request_send,ig_ncpu_neighbor,
"ig_mpi_request_send")
221 ios =
init_array_int(ig_mpi_request_recv,ig_ncpu_neighbor,
"ig_mpi_request_recv")
222 ios =
init_array_int(ig_mpi_buffer_offset,ig_ncpu_neighbor+1,
"ig_mpi_buffer_offset")
226 mpi_buffer_sizemax = 0
228 do icpu_neighbor = 1,ig_ncpu_neighbor
230 icpu = tg_cpu_neighbor(icpu_neighbor)%icpu
231 buffer_gll_duplicate(:) = 0
234 do isurf = 1,ig_nneighbor_all_kind
236 if (ig_cpu_neighbor_info(3,isurf) == icpu)
then
238 surf_num = ig_cpu_neighbor_info(2,isurf)
239 elt_num = ig_cpu_neighbor_info(1,isurf)
242 if (surf_num <= nface)
then
245 select case(surf_num)
247 num_gll = ig_hexa_gll_glonum(j,i,1,elt_num)
249 num_gll = ig_hexa_gll_glonum(j,1,i,elt_num)
251 num_gll = ig_hexa_gll_glonum(ig_ngll,j,i,elt_num)
253 num_gll = ig_hexa_gll_glonum(j,ig_ngll,i,elt_num)
255 num_gll = ig_hexa_gll_glonum(1,j,i,elt_num)
257 num_gll = ig_hexa_gll_glonum(j,i,ig_ngll,elt_num)
260 ngll_duplicate = ngll_duplicate + 1
262 if (ngll_duplicate > buffer_sizemax)
then
263 write(info,
'(a)')
"error in subroutine init_mpi_buffers: face in contact : size of buffer_gll_duplicate too small"
264 call error_stop(info)
267 buffer_gll_duplicate(ngll_duplicate) = num_gll
273 else if(surf_num <= nface+nedge)
then
275 select case(surf_num - nface)
277 num_gll = ig_hexa_gll_glonum(i,1,1,elt_num)
279 num_gll = ig_hexa_gll_glonum(ig_ngll,i,1,elt_num)
281 num_gll = ig_hexa_gll_glonum(i,ig_ngll,1,elt_num)
283 num_gll = ig_hexa_gll_glonum(1,i,1,elt_num)
285 num_gll = ig_hexa_gll_glonum(i,1,ig_ngll,elt_num)
287 num_gll = ig_hexa_gll_glonum(ig_ngll,i,ig_ngll,elt_num)
289 num_gll = ig_hexa_gll_glonum(i,ig_ngll,ig_ngll,elt_num)
291 num_gll = ig_hexa_gll_glonum(1,i,ig_ngll,elt_num)
293 num_gll = ig_hexa_gll_glonum(1,1,i,elt_num)
295 num_gll = ig_hexa_gll_glonum(ig_ngll,1,i,elt_num)
297 num_gll = ig_hexa_gll_glonum(ig_ngll,ig_ngll,i,elt_num)
299 num_gll = ig_hexa_gll_glonum(1,ig_ngll,i,elt_num)
302 ngll_duplicate = ngll_duplicate + 1
304 if (ngll_duplicate > buffer_sizemax)
then
305 write(info,
'(a)')
"error in subroutine init_mpi_buffers: edge in contact : size of buffer_gll_duplicate too small"
306 call error_stop(info)
309 buffer_gll_duplicate(ngll_duplicate) = num_gll
315 select case(surf_num - (nface+nedge))
317 num_gll = ig_hexa_gll_glonum(1,1,1,elt_num)
319 num_gll = ig_hexa_gll_glonum(ig_ngll,1,1,elt_num)
321 num_gll = ig_hexa_gll_glonum(ig_ngll,ig_ngll,1,elt_num)
323 num_gll = ig_hexa_gll_glonum(1,ig_ngll,1,elt_num)
325 num_gll = ig_hexa_gll_glonum(1,1,ig_ngll,elt_num)
327 num_gll = ig_hexa_gll_glonum(ig_ngll,1,ig_ngll,elt_num)
329 num_gll = ig_hexa_gll_glonum(ig_ngll,ig_ngll,ig_ngll,elt_num)
331 num_gll = ig_hexa_gll_glonum(1,ig_ngll,ig_ngll,elt_num)
334 ngll_duplicate = ngll_duplicate + 1
336 if (ngll_duplicate > buffer_sizemax)
then
337 write(info,
'(a)')
"error in subroutine init_mpi_buffers: node in contact : size of buffer_gll_duplicate too small"
338 call error_stop(info)
341 buffer_gll_duplicate(ngll_duplicate) = num_gll
348 call
remove_duplicate(buffer_gll_duplicate,ngll_duplicate,tg_cpu_neighbor(icpu_neighbor)%gll_send,ngll_unique)
350 if (mpi_buffer_sizemax < ngll_unique) mpi_buffer_sizemax = ngll_unique
351 tg_cpu_neighbor(icpu_neighbor)%ngll = ngll_unique
354 if (lg_async_mpi_comm)
then
355 ig_mpi_buffer_offset(icpu_neighbor+1) = ig_mpi_buffer_offset(icpu_neighbor) + tg_cpu_neighbor(icpu_neighbor)%ngll*ig_ndof
358 if (lg_output_debug_file)
then
359 open(unit=get_newunit(myunit_debug),file=
"debug."//trim(cg_prefix)//
".mpi.buffer.cpu."//trim(cg_myrank))
360 write(unit=myunit_debug,fmt=
'(3(a,i6),a)')
"from proc ",ig_myrank,
" to proc ",icpu,
" : ",ngll_unique,
" gll points in MPI buffer"
362 write(unit=myunit_debug,fmt=
'(i10,3(e14.7,1x))') tg_cpu_neighbor(icpu_neighbor)%gll_send(i)&
363 ,rg_gll_coordinate(1,tg_cpu_neighbor(icpu_neighbor)%gll_send(i))&
364 ,rg_gll_coordinate(2,tg_cpu_neighbor(icpu_neighbor)%gll_send(i))&
365 ,rg_gll_coordinate(3,tg_cpu_neighbor(icpu_neighbor)%gll_send(i))
374 do icpu_neighbor = 1,ig_ncpu_neighbor
376 icpu = tg_cpu_neighbor(icpu_neighbor)%icpu
377 ngll_send = tg_cpu_neighbor(icpu_neighbor)%ngll
379 call mpi_sendrecv(ngll_send,1,mpi_integer,icpu,89,ngll_recv,1,mpi_integer,icpu,89,mpi_comm_world,statut,ios)
381 if (ngll_send /= ngll_recv)
then
382 write(cl_rank,
'(i6.6)') icpu
383 write(info,
'(a)')
"error in subroutine init_mpi_buffers: different number ngll_send and ngll_recv betwen cpu"//trim(cg_myrank)//
" and "//trim(cl_rank)
384 call error_stop(info)
391 ig_mpi_buffer_sizemax = mpi_buffer_sizemax
396 ios =
init_array_real(gll_coord_send,ig_ncpu_neighbor,ig_ndof*mpi_buffer_sizemax,
"gll_coord_send")
398 ios =
init_array_real(gll_coord_recv,ig_ncpu_neighbor,ig_ndof*mpi_buffer_sizemax,
"gll_coord_recv")
402 do icpu_neighbor = 1,ig_ncpu_neighbor
404 icpu = tg_cpu_neighbor(icpu_neighbor)%icpu
408 do i = 1, tg_cpu_neighbor(icpu_neighbor)%ngll
409 gll_coord_send((i-1)*ig_ndof+1,icpu_neighbor) = rg_gll_coordinate(1,tg_cpu_neighbor(icpu_neighbor)%gll_send(i))
410 gll_coord_send((i-1)*ig_ndof+2,icpu_neighbor) = rg_gll_coordinate(2,tg_cpu_neighbor(icpu_neighbor)%gll_send(i))
411 gll_coord_send((i-1)*ig_ndof+3,icpu_neighbor) = rg_gll_coordinate(3,tg_cpu_neighbor(icpu_neighbor)%gll_send(i))
414 call mpi_sendrecv(gll_coord_send(1,icpu_neighbor),ig_ndof*tg_cpu_neighbor(icpu_neighbor)%ngll,mpi_real,icpu,90,gll_coord_recv(1,icpu_neighbor),ig_ndof*tg_cpu_neighbor(icpu_neighbor)%ngll,mpi_real,icpu,90,mpi_comm_world,statut,ios)
417 write(info,
'(a)')
"error in subroutine init_mpi_buffers while sending gll coordinates"
418 call error_stop(info)
425 do icpu_neighbor = 1,ig_ncpu_neighbor
426 call
create_gll_buffer_recv(gll_coord_send(1,icpu_neighbor),gll_coord_recv(1,icpu_neighbor),tg_cpu_neighbor(icpu_neighbor)%ngll,tg_cpu_neighbor(icpu_neighbor)%gll_recv,tg_cpu_neighbor(icpu_neighbor)%gll_send,mpi_buffer_sizemax)
431 deallocate(gll_coord_send)
432 deallocate(gll_coord_recv)
433 deallocate(ig_cpu_neighbor_info)
437 if (lg_async_mpi_comm)
then
439 ios =
init_array_real(rg_mpi_buffer_send,ig_mpi_buffer_offset(ig_ncpu_neighbor+1),
"rg_mpi_buffer_send")
441 ios =
init_array_real(rg_mpi_buffer_recv,ig_mpi_buffer_offset(ig_ncpu_neighbor+1),
"rg_mpi_buffer_recv")
445 if (ig_myrank == 0)
then
446 write(ig_lst_unit,
'(a)')
"done"
447 call flush(ig_lst_unit)
482 integer,
intent(in) :: ngll
483 integer,
intent(in) :: max_size
484 integer,
intent(in) ,
dimension(ngll) :: gll_send
485 integer,
intent(out),
allocatable,
dimension(:) :: gll_recv
487 real ,
intent(in),
dimension(IG_NDOF*max_size) :: gll_coord_send
488 real ,
intent(in),
dimension(IG_NDOF*max_size) :: gll_coord_recv
512 xr = gll_coord_recv((i-1)*ig_ndof+1)
513 yr = gll_coord_recv((i-1)*ig_ndof+2)
514 zr = gll_coord_recv((i-1)*ig_ndof+3)
518 xs = gll_coord_send((j-1)*ig_ndof+1)
519 ys = gll_coord_send((j-1)*ig_ndof+2)
520 zs = gll_coord_send((j-1)*ig_ndof+3)
522 dist = sqrt((xs-xr)**2 + (ys-yr)**2 + (zs-zr)**2)
524 if ( dist == 0.0 )
then
525 gll_recv(i) = gll_send(j)
527 elseif (dist < mindist)
then
534 gll_recv(i) = gll_send(myindex)
558 integer,
intent( in) :: n
559 integer,
intent( in) ,
dimension(n) :: x1
560 integer,
intent(out),
allocatable,
dimension(:) :: x3
561 integer,
intent(out) :: m
563 integer ,
dimension(n) :: x2
572 if (x2(j) == x1(i))
then
This module contains subroutines to allocate arrays and to compute an approximation of the total RAM ...
subroutine, public init_mpi_buffers()
This subroutine searches for common GLL nodes between cpu myrank and its neighbor cpus...
This module defines all global variables of EFISPEC3D. Scalar variables are initialized directly in t...
Interface init_array_real to redirect allocation to n-rank arrays.
subroutine, private remove_duplicate(x1, n, x3, m)
This subroutine removes duplicated GLLs between cpu myrank and its connected cpus.
subroutine, private create_gll_buffer_recv(gll_coord_send, gll_coord_recv, ngll, gll_recv, gll_send, max_size)
This subroutine creates GLLs buffers received by cpu myrank from other cpus. Identical GLLs between c...
This module contains subroutines to initialize MPI buffers between cpu myrank and its neighbor cpus...
Interface init_array_int to redirect allocation to n-rank arrays.