public Comm(SPARTA sparta)
{
this.sparta = sparta;
sparta.mpi.MPI_Comm_rank(sparta.world, ref me);
sparta.mpi.MPI_Comm_size(sparta.world, ref nprocs);
ncomm = 0;
commsortflag = 0;
commpartstyle = 1;
neighflag = 0;
neighlist = null;
iparticle = new Irregular(sparta);
igrid = null;
iuniform = null;
pproc = null;
maxpproc = 0;
gproc = gsize = null;
maxgproc = 0;
sbuf = rbuf = null;
maxsendbuf = maxrecvbuf = 0;
copymode = 0;
}
public RCB(SPARTA sparta)
{
this.sparta = sparta;
sparta.mpi.MPI_Comm_rank(sparta.world, ref me);
sparta.mpi.MPI_Comm_size(sparta.world, ref nprocs);
ndot = maxdot = 0;
dots = null;
nlist = maxlist = 0;
dotlist = dotmark = null;
maxbuf = 0;
buf = null;
maxrecv = maxsend = 0;
recvproc = recvindex = sendproc = sendindex = null;
//tree = (Tree*)memory->smalloc(nprocs * sizeof(Tree), "RCB:tree");
tree = new Tree[nprocs];
irregular = null;
// create MPI data and function types for box and median AllReduce ops
sparta.mpi.MPI_Type_contiguous(6, MPI.MPI_DOUBLE, ref box_type);
sparta.mpi.MPI_Type_commit(ref box_type);
sparta.mpi.MPI_Type_contiguous(Marshal.SizeOf(typeof(Median)), MPI.MPI_CHAR, ref med_type);
sparta.mpi.MPI_Type_commit(ref med_type);
sparta.mpi.MPI_Op_create(box_merge, 1, ref box_op);
sparta.mpi.MPI_Op_create(median_merge, 1, ref med_op);
reuse = 0;
}
public virtual void migrate_cells(int nmigrate)
{
int i, n;
Grid.ChildCell[] cells = sparta.grid.cells;
int nglocal = sparta.grid.nlocal;
// grow proc and size lists if needed
if (nmigrate > maxgproc)
{
maxgproc = nmigrate;
//memory.destroy(gproc);
//memory.destroy(gsize);
gproc = new int[maxgproc];
gsize = new int[maxgproc];
//memory.create(gproc, maxgproc, "comm:gproc");
//memory.create(gsize, maxgproc, "comm:gsize");
}
// fill proclist with procs to send to
// compute byte count needed to pack cells
int nsend = 0;
bigint boffset = 0;
StringBuilder sb = new StringBuilder();
for (int icell = 0; icell < nglocal; icell++)
{
if (cells[icell].nsplit <= 0)
{
continue;
}
if (cells[icell].proc == me)
{
continue;
}
gproc[nsend] = cells[icell].proc;
n = sparta.grid.pack_one(icell, 1, 1, 0, ref sb);
gsize[nsend++] = n;
boffset += n;
}
if (boffset > Run.MAXSMALLINT)
{
sparta.error.one("Migrate cells send buffer exceeds 2 GB");
}
int offset = (int)boffset;
// reallocate sbuf as needed
if (offset > maxsendbuf)
{
//memory.destroy(sbuf);
maxsendbuf = offset;
sbuf = new byte[maxsendbuf];
//memory.create(sbuf, maxsendbuf, "comm:sbuf");
//memset(sbuf, 0, maxsendbuf);
}
sbuf = new byte[maxsendbuf];
rbuf = new byte[maxsendbuf];
// pack cell info into sbuf
// only called for unsplit and split cells I no longer own
offset = 0;
for (int icell = 0; icell < nglocal; icell++)
{
if (cells[icell].nsplit <= 0)
{
continue;
}
if (cells[icell].proc == me)
{
continue;
}
char[] tmp = new char[maxsendbuf - offset];
Array.Copy(sbuf, offset, tmp, 0, maxsendbuf - offset);
StringBuilder sb1 = new StringBuilder($"{sbuf}");
offset += sparta.grid.pack_one(icell, 1, 1, 1, ref sb1);
}
// compress my list of owned grid cells to remove migrated cells
// compress particle list to remove particles in migrating cells
// unset particle sorted since compress_rebalance() does
// and may receive particles which will then be unsorted
if (nmigrate != 0)
{
sparta.grid.compress();
sparta.particle.compress_rebalance();
}
else
{
sparta.particle.sorted = 0;
}
// create irregular communication plan with variable size datums
// nrecv = # of incoming grid cells
// recvsize = total byte size of incoming grid + particle info
// DEBUG: append a sort=1 arg so that messages from other procs
// are received in repeatable order, thus grid cells stay in order
if (igrid == null)
{
igrid = new Irregular(sparta);
}
int recvsize;
int nrecv =
igrid.create_data_variable(nmigrate, gproc, gsize, out recvsize, commsortflag);
// reallocate rbuf as needed
if (recvsize > maxrecvbuf)
{
// memory.destroy(rbuf);
maxrecvbuf = recvsize;
rbuf = new byte[maxrecvbuf];
//memory.create(rbuf, maxrecvbuf, "comm:rbuf");
//memset(rbuf, 0, maxrecvbuf);
}
// perform irregular communication
byte[] bytes = new byte[sbuf.Length * sizeof(char)];
bytes = Encoding.UTF8.GetBytes(sbuf.ToString());
byte[] byter = new byte[rbuf.Length * sizeof(char)];
byter = Encoding.UTF8.GetBytes(rbuf.ToString());
igrid.exchange_variable(bytes, gsize, byter);
// unpack received grid cells with their particles
offset = 0;
for (i = 0; i < nrecv; i++)
{
offset += sparta.grid.unpack_one(new StringBuilder(), 1, 1);
}
Console.Write("comm.migrate_cells().unpackone\n");
for (int a = offset; a < rbuf.Length - 1 - offset; a++)
{
Console.Write("{0}", rbuf[a]);
}
}
public void invert()
{
if (irregular == null)
{
irregular = new Irregular(sparta);
}
// nsend = # of dots to request from other procs
int nsend = nfinal - nkeep;
int[] proclist;
proclist = new int[nsend];
Invert[] sinvert = new Invert[nsend];
int m = 0;
for (int i = nkeep; i < nfinal; i++)
{
proclist[m] = recvproc[i];
sinvert[m].rindex = recvindex[i];
sinvert[m].sproc = me;
sinvert[m].sindex = i;
m++;
}
// perform inversion via irregular comm
// nrecv = # of my dots to send to other procs
int nrecv = irregular.create_data_uniform(nsend, proclist, sparta.comm.commsortflag);
Invert[] rinvert = new Invert[nrecv];
irregular.exchange_uniform(sinvert.ToString(), Marshal.SizeOf(typeof(Invert)), rinvert.ToString());
// set public variables from requests to send my dots
if (noriginal > maxsend)
{
maxsend = noriginal;
sendproc = new int[maxsend];
sendindex = new int[maxsend];
//memory->create(sendproc, maxsend, "RCB:sendproc");
//memory->create(sendindex, maxsend, "RCB:sendindex");
}
for (int i = 0; i < nkeep; i++)
{
sendproc[recvindex[i]] = me;
sendindex[recvindex[i]] = i;
}
for (int i = 0; i < nrecv; i++)
{
m = rinvert[i].rindex;
sendproc[m] = rinvert[i].sproc;
sendindex[m] = rinvert[i].sindex;
}
// clean-up
//memory->destroy(proclist);
//memory->destroy(sinvert);
//memory->destroy(rinvert);
}