Newer
Older
/*----------------------------------------------------------------------
PuReMD - Purdue ReaxFF Molecular Dynamics Program
Copyright (2010) Purdue University
Hasan Metin Aktulga, haktulga@cs.purdue.edu
Joseph Fogarty, jcfogart@mail.usf.edu
Sagar Pandit, pandit@usf.edu
Ananth Y Grama, ayg@cs.purdue.edu
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
See the GNU General Public License for more details:
<http://www.gnu.org/licenses/>.
----------------------------------------------------------------------*/
#include "comm_tools.h"
#include "grid.h"
#include "reset_tools.h"
#include "tool_box.h"
#include "vector.h"
Kurt A. O'Hearn
committed
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
#if defined(NEUTRAL_TERRITORY)
void Setup_NT_Comm( reax_system * const system, control_params * const control,
mpi_datatypes * const mpi_data )
{
int i, d;
real bndry_cut;
neighbor_proc *nbr_pr;
simulation_box *my_box;
ivec nbr_coords, nbr_recv_coords;
ivec r[12] = {
{0, 0, -1}, // -z
{0, 0, +1}, // +z
{0, -1, 0}, // -y
{-1, -1, 0}, // -x-y
{-1, 0, 0}, // -x
{-1, +1, 0}, // -x+y
{0, 0, +1}, // +z
{0, 0, -1}, // -z
{0, +1, 0}, // +y
{+1, +1, 0}, // +x+y
{+1, 0, 0}, // +x
{+1, -1, 0} // +x-y
};
my_box = &system->my_box;
bndry_cut = system->bndry_cuts.ghost_cutoff;
system->num_nt_nbrs = MAX_NT_NBRS;
/* identify my neighbors */
for ( i = 0; i < system->num_nt_nbrs; ++i )
{
nbr_pr = &system->my_nt_nbrs[i];
ivec_Sum( nbr_coords, system->my_coords, r[i] ); /* actual nbr coords */
MPI_Cart_rank( mpi_data->comm_mesh3D, nbr_coords, &nbr_pr->rank );
/* set the rank of the neighbor processor in the receiving direction */
ivec_Sum( nbr_recv_coords, system->my_coords, r[i + 6] ); /* actual nbr coords */
MPI_Cart_rank( mpi_data->comm_mesh3D, nbr_recv_coords, &nbr_pr->receive_rank );
for ( d = 0; d < 3; ++d )
{
/* determine the boundary area with this nbr */
if ( r[i][d] < 0 )
{
nbr_pr->bndry_min[d] = my_box->min[d];
nbr_pr->bndry_max[d] = my_box->min[d] + bndry_cut;
}
else if ( r[i][d] > 0 )
{
nbr_pr->bndry_min[d] = my_box->max[d] - bndry_cut;
nbr_pr->bndry_max[d] = my_box->max[d];
}
else
{
nbr_pr->bndry_min[d] = my_box->min[d];
nbr_pr->bndry_max[d] = my_box->max[d];
}
/* determine if it is a periodic neighbor */
if ( nbr_coords[d] < 0 )
{
nbr_pr->prdc[d] = -1;
}
else if ( nbr_coords[d] >= control->procs_by_dim[d] )
{
nbr_pr->prdc[d] = 1;
}
else
{
nbr_pr->prdc[d] = 0;
}
}
}
}
static int Sort_Neutral_Territory( reax_system *system, int dir, mpi_out_data *out_bufs, int write )
{
int i, cnt;
reax_atom *atoms;
neighbor_proc *nbr_pr;
cnt = 0;
atoms = system->my_atoms;
/* place each atom into the appropriate outgoing list */
Kurt A. O'Hearn
committed
nbr_pr = &system->my_nt_nbrs[dir];
Kurt A. O'Hearn
committed
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
for ( i = 0; i < system->n; ++i )
{
if ( nbr_pr->bndry_min[0] <= atoms[i].x[0]
&& atoms[i].x[0] < nbr_pr->bndry_max[0]
&& nbr_pr->bndry_min[1] <= atoms[i].x[1]
&& atoms[i].x[1] < nbr_pr->bndry_max[1]
&& nbr_pr->bndry_min[2] <= atoms[i].x[2]
&& atoms[i].x[2] < nbr_pr->bndry_max[2] )
{
if ( write )
{
out_bufs[dir].index[out_bufs[dir].cnt] = i;
out_bufs[dir].cnt++;
}
else
{
cnt++;
}
}
}
return cnt;
}
void Estimate_NT_Atoms( reax_system * const system, mpi_datatypes * const mpi_data )
{
int d;
mpi_out_data *out_bufs;
neighbor_proc *nbr;
out_bufs = mpi_data->out_nt_buffers;
for ( d = 0; d < 6; ++d )
{
/* count the number of atoms in each processor's outgoing list */
nbr = &system->my_nt_nbrs[d];
nbr->est_send = Sort_Neutral_Territory( system, d, out_bufs, 0 );
/* estimate the space needed based on the count above */
nbr->est_send = MAX( MIN_SEND, nbr->est_send * SAFER_ZONE_NT );
/* allocate the estimated space */
out_bufs[d].index = scalloc( 2 * nbr->est_send, sizeof(int),
__FILE__, __LINE__ );
out_bufs[d].out_atoms = scalloc( 2 * nbr->est_send, sizeof(real),
__FILE__, __LINE__ );
Kurt A. O'Hearn
committed
/* sort the atoms to their outgoing buffers */
// TODO: to call or not to call?
//Sort_Neutral_Territory( system, d, out_bufs, 1 );
}
}
#endif
Kurt A. O'Hearn
committed
/* Note: filename must be NULL-terminated before calling this function */
Kurt A. O'Hearn
committed
void Check_MPI_Error( int code, const char * const filename, int line )
Kurt A. O'Hearn
committed
char err_msg[MPI_MAX_ERROR_STRING];
if ( code != MPI_SUCCESS )
{
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
MPI_Error_string( code, err_msg, &len );
Kurt A. O'Hearn
committed
fprintf( stderr, "[ERROR] MPI error\n" );
Kurt A. O'Hearn
committed
/* strlen safe here only if filename is NULL-terminated before calling Check_MPI_Error */
fprintf( stderr, " [INFO] At line %d in file %.*s on MPI processor %d\n",
line, (int) strlen(filename), filename, rank );
fprintf( stderr, " [INFO] Error code: %d\n", code );
Kurt A. O'Hearn
committed
fprintf( stderr, " [INFO] Error message: %.*s\n", len, err_msg );
Kurt A. O'Hearn
committed
MPI_Abort( MPI_COMM_WORLD, RUNTIME_ERROR );
Kurt A. O'Hearn
committed
void Setup_Comm( reax_system * const system, control_params * const control,
mpi_datatypes * const mpi_data )
Kurt A. O'Hearn
committed
const real bndry_cut = system->bndry_cuts.ghost_cutoff;
Kurt A. O'Hearn
committed
simulation_box * const my_box = &system->my_box;
{ -1, 0, 0}, { 1, 0, 0}, // -x, +x
{0, -1, 0}, {0, 1, 0}, // -y, +y
{0, 0, -1}, {0, 0, 1}, // -z, +z
/* identify my neighbors */
system->num_nbrs = MAX_NBRS;
for ( i = 0; i < system->num_nbrs; ++i )
{
ivec_Sum( nbr_coords, system->my_coords, r[i] ); /* actual nbr coords */
nbr_pr = &system->my_nbrs[i];
Kurt A. O'Hearn
committed
MPI_Cart_rank( mpi_data->comm_mesh3D, nbr_coords, &nbr_pr->rank );
for ( d = 0; d < 3; ++d )
{
/* determine the boundary area with this nbr */
if ( r[i][d] < 0 )
{
nbr_pr->bndry_min[d] = my_box->min[d];
nbr_pr->bndry_max[d] = my_box->min[d] + bndry_cut;
}
else if ( r[i][d] > 0 )
{
nbr_pr->bndry_min[d] = my_box->max[d] - bndry_cut;
nbr_pr->bndry_max[d] = my_box->max[d];
}
else
{
Kurt A. O'Hearn
committed
nbr_pr->bndry_min[d] = NEG_INF;
nbr_pr->bndry_max[d] = NEG_INF;
}
/* determine if it is a periodic neighbor */
if ( nbr_coords[d] < 0 )
else if ( nbr_coords[d] >= control->procs_by_dim[d] )
Kurt A. O'Hearn
committed
nbr_pr->prdc[d] = 1;
Kurt A. O'Hearn
committed
void Update_Comm( reax_system * const system )
Kurt A. O'Hearn
committed
const real bndry_cut = system->bndry_cuts.ghost_cutoff;
Kurt A. O'Hearn
committed
simulation_box * const my_box = &system->my_box;
{ -1, 0, 0}, { 1, 0, 0}, // -x, +x
{0, -1, 0}, {0, 1, 0}, // -y, +y
{0, 0, -1}, {0, 0, 1}, // -z, +z
/* identify my neighbors */
for ( i = 0; i < system->num_nbrs; ++i )
{
nbr_pr = &system->my_nbrs[i];
/* determine the boundary area with this nbr */
if ( r[i][d] < 0 )
{
nbr_pr->bndry_min[d] = my_box->min[d];
nbr_pr->bndry_max[d] = my_box->min[d] + bndry_cut;
}
else if ( r[i][d] > 0 )
{
nbr_pr->bndry_min[d] = my_box->max[d] - bndry_cut;
nbr_pr->bndry_max[d] = my_box->max[d];
}
else
{
Kurt A. O'Hearn
committed
nbr_pr->bndry_min[d] = NEG_INF;
nbr_pr->bndry_max[d] = NEG_INF;
}
/********************* ATOM TRANSFER ***********************/
/***************** PACK & UNPACK ATOMS *********************/
Kurt A. O'Hearn
committed
static void Pack_MPI_Atom( mpi_atom * const matm, const reax_atom * const ratm, int i )
matm->orig_id = ratm->orig_id;
Kurt A. O'Hearn
committed
matm->imprt_id = i;
matm->type = ratm->type;
Kurt A. O'Hearn
committed
matm->num_bonds = ratm->num_bonds;
matm->num_hbonds = ratm->num_hbonds;
Kurt A. O'Hearn
committed
strncpy( matm->name, ratm->name, sizeof(matm->name) - 1 );
matm->name[sizeof(matm->name) - 1] = '\0';
rvec_Copy( matm->x, ratm->x );
rvec_Copy( matm->v, ratm->v );
rvec_Copy( matm->f_old, ratm->f_old );
memcpy( matm->s, ratm->s, sizeof(rvec4) ); //rvec_Copy( matm->s, ratm->s );
memcpy( matm->t, ratm->t, sizeof(rvec4) ); //rvec_Copy( matm->t, ratm->t );
Kurt A. O'Hearn
committed
static void Unpack_MPI_Atom( reax_atom * const ratm, const mpi_atom * const matm )
ratm->orig_id = matm->orig_id;
Kurt A. O'Hearn
committed
ratm->imprt_id = matm->imprt_id;
ratm->type = matm->type;
Kurt A. O'Hearn
committed
ratm->num_bonds = matm->num_bonds;
ratm->num_hbonds = matm->num_hbonds;
Kurt A. O'Hearn
committed
strncpy( ratm->name, matm->name, sizeof(ratm->name) - 1 );
ratm->name[sizeof(ratm->name) - 1] = '\0';
rvec_Copy( ratm->x, matm->x );
rvec_Copy( ratm->v, matm->v );
rvec_Copy( ratm->f_old, matm->f_old );
memcpy( ratm->s, matm->s, sizeof(rvec4) ); //rvec_Copy( ratm->s, matm->s );
memcpy( ratm->t, matm->t, sizeof(rvec4) ); //rvec_Copy( ratm->t, matm->t );
Kurt A. O'Hearn
committed
/* Count number of atoms to be placed in egress MPI buffers.
* The ownership of these atoms is to be transferred to other MPI processes. */
static void Count_Transfer_Atoms( reax_system const * const system,
int start, int end, int dim, mpi_out_data * const out_bufs,
Kurt A. O'Hearn
committed
int * const cnt )
Kurt A. O'Hearn
committed
{
int i, d;
Kurt A. O'Hearn
committed
simulation_box const * const my_box = &system->my_box;
Kurt A. O'Hearn
committed
/* count atoms in the appropriate outgoing lists */
for ( i = start; i < end; ++i )
{
for ( d = dim; d < 3; ++d )
{
if ( system->my_atoms[i].x[d] < my_box->min[d] )
{
Kurt A. O'Hearn
committed
++cnt[2 * d];
Kurt A. O'Hearn
committed
break;
}
else if ( system->my_atoms[i].x[d] >= my_box->max[d] )
{
Kurt A. O'Hearn
committed
++cnt[2 * d + 1];
Kurt A. O'Hearn
committed
break;
}
}
}
}
/* Populate egress MPI buffers with atoms (of reax_atom type) whose
* ownership is being transferred to other MPI processes
*
* Note: the "sort" refers to dividing the atoms into their appropriate buffers */
Kurt A. O'Hearn
committed
static void Sort_Transfer_Atoms( reax_system * const system, int start, int end,
Kurt A. O'Hearn
committed
int dim, mpi_out_data * const out_bufs, mpi_datatypes * const mpi_data )
Kurt A. O'Hearn
committed
int i, d;
Kurt A. O'Hearn
committed
simulation_box * const my_box = &system->my_box;
Kurt A. O'Hearn
committed
/* place each atom into the appropriate egress buffer */
for ( i = start; i < end; ++i )
{
for ( d = dim; d < 3; ++d )
Kurt A. O'Hearn
committed
if ( system->my_atoms[i].x[d] < my_box->min[d] )
Kurt A. O'Hearn
committed
out_buf = out_bufs[2 * d].out_atoms;
Kurt A. O'Hearn
committed
Pack_MPI_Atom( &out_buf[out_bufs[2 * d].cnt], &system->my_atoms[i], i );
Kurt A. O'Hearn
committed
system->my_atoms[i].orig_id = -1;
Kurt A. O'Hearn
committed
++out_bufs[2 * d].cnt;
Kurt A. O'Hearn
committed
else if ( system->my_atoms[i].x[d] >= my_box->max[d] )
Kurt A. O'Hearn
committed
out_buf = out_bufs[2 * d + 1].out_atoms;
Kurt A. O'Hearn
committed
Pack_MPI_Atom( &out_buf[out_bufs[2 * d + 1].cnt], &system->my_atoms[i], i );
Kurt A. O'Hearn
committed
system->my_atoms[i].orig_id = -1;
Kurt A. O'Hearn
committed
++out_bufs[2 * d + 1].cnt;
}
/*********************** UNPACKER **************************/
Kurt A. O'Hearn
committed
static void Unpack_Transfer_Message( reax_system * const system,
int end, void * const mpi_buffer,
Kurt A. O'Hearn
committed
int cnt, neighbor_proc * const nbr, int dim )
Kurt A. O'Hearn
committed
mpi_atom * const src = (mpi_atom *) mpi_buffer;
Kurt A. O'Hearn
committed
if ( end + cnt > system->total_cap )
Kurt A. O'Hearn
committed
/* need space for my_atoms now, other reallocations will trigger in Reallocate */
Kurt A. O'Hearn
committed
system->my_atoms = srealloc_pinned( system->my_atoms,
sizeof(reax_atom) * MAX( system->total_cap, end ),
sizeof(reax_atom) * (end + cnt), __FILE__, __LINE__ );
/* adjust coordinates of recved atoms if nbr is a periodic one */
if ( nbr->prdc[dim] )
{
dx = nbr->prdc[dim] * system->big_box.box_norms[dim];
Kurt A. O'Hearn
committed
Kurt A. O'Hearn
committed
Unpack_MPI_Atom( &system->my_atoms[end + i], &src[i] );
Kurt A. O'Hearn
committed
system->my_atoms[end + i].x[dim] += dx;
Kurt A. O'Hearn
committed
else
{
for ( i = 0; i < cnt; ++i )
{
Unpack_MPI_Atom( &system->my_atoms[end + i], &src[i] );
}
}
}
/************** EXCHANGE BOUNDARY ATOMS *****************/
/************ PACK & UNPACK BOUNDARY ATOMS **************/
Kurt A. O'Hearn
committed
static void Pack_Boundary_Atom( boundary_atom * const matm,
Kurt A. O'Hearn
committed
reax_atom const * const ratm, int i )
matm->orig_id = ratm->orig_id;
Kurt A. O'Hearn
committed
matm->imprt_id = i;
matm->type = ratm->type;
Kurt A. O'Hearn
committed
matm->num_bonds = ratm->num_bonds;
matm->num_hbonds = ratm->num_hbonds;
Kurt A. O'Hearn
committed
static void Unpack_Boundary_Atom( reax_atom * const ratm,
const boundary_atom * const matm )
Kurt A. O'Hearn
committed
ratm->imprt_id = matm->imprt_id;
Kurt A. O'Hearn
committed
ratm->num_bonds = matm->num_bonds;
ratm->num_hbonds = matm->num_hbonds;
// ratm->renumber = offset + matm->imprt_id;
Kurt A. O'Hearn
committed
void Count_Boundary_Atoms( reax_system const * const system, int start, int end,
int dim, mpi_out_data * const out_bufs, int * const cnt )
Kurt A. O'Hearn
committed
int i, d, p;
/* place each atom into the appropriate outgoing list */
Kurt A. O'Hearn
committed
for ( i = start; i < end; ++i )
Kurt A. O'Hearn
committed
for ( d = dim; d < 3; ++d )
Kurt A. O'Hearn
committed
{
Kurt A. O'Hearn
committed
for ( p = 2 * d; p < 2 * d + 2; ++p )
{
if ( system->my_nbrs[p].bndry_min[d] <= system->my_atoms[i].x[d]
&& system->my_atoms[i].x[d] < system->my_nbrs[p].bndry_max[d] )
{
++cnt[p];
}
}
Kurt A. O'Hearn
committed
}
Kurt A. O'Hearn
committed
void Sort_Boundary_Atoms( reax_system * const system, int start, int end,
int dim, mpi_out_data * const out_bufs, mpi_datatypes * const mpi_data )
Kurt A. O'Hearn
committed
int i, d, p;
Kurt A. O'Hearn
committed
/* place each atom into the appropriate egress buffer */
Kurt A. O'Hearn
committed
for ( i = start; i < end; ++i )
Kurt A. O'Hearn
committed
for ( d = dim; d < 3; ++d )
Kurt A. O'Hearn
committed
for ( p = 2 * d; p < 2 * d + 2; ++p )
Kurt A. O'Hearn
committed
if ( system->my_nbrs[p].bndry_min[d] <= system->my_atoms[i].x[d]
&& system->my_atoms[i].x[d] < system->my_nbrs[p].bndry_max[d] )
{
out_bufs[p].index[out_bufs[p].cnt] = i;
out_buf = (boundary_atom *) out_bufs[p].out_atoms;
Pack_Boundary_Atom( &out_buf[out_bufs[p].cnt], &system->my_atoms[i], i );
++out_bufs[p].cnt;
}
Kurt A. O'Hearn
committed
/* Copy received atoms out of MPI ingress buffer */
void Unpack_Exchange_Message( reax_system * const system, int end,
void * const mpi_buffer, int cnt,
neighbor_proc * const nbr, int dim )
Kurt A. O'Hearn
committed
const boundary_atom * const src = (boundary_atom *) mpi_buffer;
Kurt A. O'Hearn
committed
if ( end + cnt > system->total_cap )
Kurt A. O'Hearn
committed
/* need space for my_atoms now, other reallocations will trigger in Reallocate */
Kurt A. O'Hearn
committed
system->my_atoms = srealloc_pinned( system->my_atoms,
sizeof(reax_atom) * MAX( system->total_cap, end ),
sizeof(reax_atom) * (end + cnt), __FILE__, __LINE__ );
Kurt A. O'Hearn
committed
/* adjust coordinates of recved atoms if nbr is a periodic one */
dx = nbr->prdc[dim] * system->big_box.box_norms[dim];
Kurt A. O'Hearn
committed
Unpack_Boundary_Atom( &system->my_atoms[end + i], &src[i] );
system->my_atoms[end + i].x[dim] += dx;
}
}
else
{
for ( i = 0; i < cnt; ++i )
{
Unpack_Boundary_Atom( &system->my_atoms[end + i], &src[i] );
Kurt A. O'Hearn
committed
/* record the atoms recv'd from this nbr */
nbr->atoms_str = end;
nbr->atoms_cnt = cnt;
Kurt A. O'Hearn
committed
static void Count_Position_Updates( reax_system const * const system,
int start, int end, int dim, mpi_out_data * const out_bufs,
Kurt A. O'Hearn
committed
int * const cnt )
Kurt A. O'Hearn
committed
int p;
/* counts set via previous calls to SendRecv in reneighbor steps
* (during boundary atom messaging), so just copy */
for ( p = 2 * dim; p < 2 * dim + 2; ++p )
{
cnt[p] = out_bufs[p].cnt;
}
Kurt A. O'Hearn
committed
static void Sort_Position_Updates( reax_system * const system, int start, int end,
Kurt A. O'Hearn
committed
int dim, mpi_out_data * const out_bufs, mpi_datatypes * const mpi_data )
Kurt A. O'Hearn
committed
out = (rvec *) out_bufs[p].out_atoms;
Kurt A. O'Hearn
committed
rvec_Copy( out[i], system->my_atoms[ out_bufs[p].index[i] ].x );
Kurt A. O'Hearn
committed
static void Unpack_Position_Updates( reax_system * const system, int end,
void * const mpi_buffer, int cnt, neighbor_proc * const nbr, int dim )
Kurt A. O'Hearn
committed
int i;
const int start = nbr->atoms_str;
Kurt A. O'Hearn
committed
rvec * const src = (rvec *) mpi_buffer;
Kurt A. O'Hearn
committed
/* adjust coordinates of recved atoms if nbr is a periodic one */
dx = nbr->prdc[dim] * system->big_box.box_norms[dim];
Kurt A. O'Hearn
committed
Kurt A. O'Hearn
committed
rvec_Copy( system->my_atoms[start + i].x, src[i] );
system->my_atoms[start + i].x[dim] += dx;
}
}
else
{
for ( i = 0; i < cnt; ++i )
{
rvec_Copy( system->my_atoms[start + i].x, src[i] );
Kurt A. O'Hearn
committed
int SendRecv( reax_system * const system, mpi_datatypes * const mpi_data,
MPI_Datatype type, message_counter count_func,
message_sorter sort_func, unpacker unpack, int clr )
Kurt A. O'Hearn
committed
int i, d, cnt1, cnt2, cnt[6], start, end, ret;
Kurt A. O'Hearn
committed
mpi_out_data * const out_bufs = mpi_data->out_buffers;
const MPI_Comm comm = mpi_data->comm_mesh3D;
MPI_Request req1, req2;
MPI_Status stat1, stat2;
neighbor_proc *nbr1, *nbr2;
Kurt A. O'Hearn
committed
MPI_Aint extent, lower_bound;
size_t type_size;
Kurt A. O'Hearn
committed
Kurt A. O'Hearn
committed
ret = MPI_Type_get_extent( type, &lower_bound, &extent );
Check_MPI_Error( ret, __FILE__, __LINE__ );
// type_size = MPI_Aint_add( lower_bound, extent );
type_size = extent;
if ( clr == TRUE )
{
Reset_Out_Buffers( mpi_data->out_buffers, system->num_nbrs );
}
start = 0;
end = system->n;
for ( d = 0; d < 3; ++d )
{
Kurt A. O'Hearn
committed
/* nbr1 is in the negative direction, nbr2 the positive direction */
Kurt A. O'Hearn
committed
nbr1 = &system->my_nbrs[2 * d];
nbr2 = &system->my_nbrs[2 * d + 1];
Kurt A. O'Hearn
committed
for ( i = 0; i < 6; ++i )
{
cnt[i] = 0;
}
Kurt A. O'Hearn
committed
count_func( system, start, end, d, out_bufs, cnt );
Kurt A. O'Hearn
committed
for ( i = 2 * d; i < 6; ++i )
Kurt A. O'Hearn
committed
srealloc_check( &out_bufs[i].out_atoms,
Kurt A. O'Hearn
committed
&out_bufs[i].out_atoms_size,
type_size * (out_bufs[i].cnt + cnt[i]),
TRUE, SAFE_ZONE, __FILE__, __LINE__ );
Kurt A. O'Hearn
committed
srealloc_check( (void **) &out_bufs[i].index,
Kurt A. O'Hearn
committed
&out_bufs[i].index_size,
sizeof(int) * (out_bufs[i].cnt + cnt[i]),
TRUE, SAFE_ZONE, __FILE__, __LINE__ );
Kurt A. O'Hearn
committed
sort_func( system, start, end, d, out_bufs, mpi_data );
Kurt A. O'Hearn
committed
start = end;
Kurt A. O'Hearn
committed
ret = MPI_Isend( out_bufs[2 * d].out_atoms, out_bufs[2 * d].cnt,
type, nbr1->rank, 2 * d, comm, &req1 );
Kurt A. O'Hearn
committed
Check_MPI_Error( ret, __FILE__, __LINE__ );
Kurt A. O'Hearn
committed
Kurt A. O'Hearn
committed
ret = MPI_Isend( out_bufs[2 * d + 1].out_atoms, out_bufs[2 * d + 1].cnt,
type, nbr2->rank, 2 * d + 1, comm, &req2 );
Kurt A. O'Hearn
committed
Check_MPI_Error( ret, __FILE__, __LINE__ );
/* recv and unpack atoms from nbr1 in dimension d */
Kurt A. O'Hearn
committed
ret = MPI_Probe( nbr1->rank, 2 * d + 1, comm, &stat1 );
Kurt A. O'Hearn
committed
Check_MPI_Error( ret, __FILE__, __LINE__ );
Kurt A. O'Hearn
committed
ret = MPI_Get_count( &stat1, type, &cnt1 );
Kurt A. O'Hearn
committed
Check_MPI_Error( ret, __FILE__, __LINE__ );
Kurt A. O'Hearn
committed
if ( cnt1 == MPI_UNDEFINED )
{
fprintf( stderr, "[ERROR] MPI_Get_count returned MPI_UNDEFINED\n" );
MPI_Abort( MPI_COMM_WORLD, RUNTIME_ERROR );
}
Kurt A. O'Hearn
committed
smalloc_check( &mpi_data->in1_buffer, &mpi_data->in1_buffer_size,
type_size * cnt1, TRUE, SAFE_ZONE, __FILE__, __LINE__ );
Kurt A. O'Hearn
committed
ret = MPI_Recv( mpi_data->in1_buffer, cnt1, type,
Kurt A. O'Hearn
committed
nbr1->rank, 2 * d + 1, comm, MPI_STATUS_IGNORE );
Kurt A. O'Hearn
committed
Check_MPI_Error( ret, __FILE__, __LINE__ );
Kurt A. O'Hearn
committed
unpack( system, end, mpi_data->in1_buffer, cnt1, nbr1, d );
end += cnt1;
Kurt A. O'Hearn
committed
ret = MPI_Probe( nbr2->rank, 2 * d, comm, &stat2 );
Kurt A. O'Hearn
committed
Check_MPI_Error( ret, __FILE__, __LINE__ );
Kurt A. O'Hearn
committed
ret = MPI_Get_count( &stat2, type, &cnt2 );
Kurt A. O'Hearn
committed
Check_MPI_Error( ret, __FILE__, __LINE__ );
Kurt A. O'Hearn
committed
if ( cnt2 == MPI_UNDEFINED )
{
fprintf( stderr, "[ERROR] MPI_Get_count returned MPI_UNDEFINED\n" );
MPI_Abort( MPI_COMM_WORLD, RUNTIME_ERROR );
}
Kurt A. O'Hearn
committed
smalloc_check( &mpi_data->in2_buffer, &mpi_data->in2_buffer_size,
type_size * cnt2, TRUE, SAFE_ZONE, __FILE__, __LINE__ );
Kurt A. O'Hearn
committed
ret = MPI_Recv( mpi_data->in2_buffer, cnt2, type,
Kurt A. O'Hearn
committed
nbr2->rank, 2 * d, comm, MPI_STATUS_IGNORE );
Kurt A. O'Hearn
committed
Check_MPI_Error( ret, __FILE__, __LINE__ );
Kurt A. O'Hearn
committed
unpack( system, end, mpi_data->in2_buffer, cnt2, nbr2, d );
end += cnt2;
ret = MPI_Wait( &req1, MPI_STATUS_IGNORE );
Check_MPI_Error( ret, __FILE__, __LINE__ );
ret = MPI_Wait( &req2, MPI_STATUS_IGNORE );
Check_MPI_Error( ret, __FILE__, __LINE__ );
Kurt A. O'Hearn
committed
void Comm_Atoms( reax_system * const system, control_params * const control,
simulation_data * const data, storage * const workspace,
mpi_datatypes * const mpi_data, int renbr )
Kurt A. O'Hearn
committed
real time;
Kurt A. O'Hearn
committed
time = Get_Time( );
if ( renbr == TRUE )
Kurt A. O'Hearn
committed
/* transfer ownership of any previous local atoms
* which have moved outside of this processor's local sim. box */
system->n = SendRecv( system, mpi_data, mpi_data->mpi_atom_type,
&Count_Transfer_Atoms, &Sort_Transfer_Atoms,
&Unpack_Transfer_Message, TRUE );
Bin_My_Atoms( system, workspace );
Kurt A. O'Hearn
committed
/* exchange ghost region info with neighbors */
Kurt A. O'Hearn
committed
system->N = SendRecv( system, mpi_data, mpi_data->boundary_atom_type,
&Count_Boundary_Atoms, &Sort_Boundary_Atoms,
&Unpack_Exchange_Message, TRUE );
Kurt A. O'Hearn
committed
/* provide position updates of atoms to neighboring processors */
SendRecv( system, mpi_data, mpi_data->mpi_rvec,
&Count_Position_Updates, &Sort_Position_Updates,
&Unpack_Position_Updates, FALSE );
Kurt A. O'Hearn
committed
Update_Timing_Info( &time, &data->timing.comm );