/// <summary>
/// Adopts a low-level MPI group that was created with any of the low-level MPI facilities.
/// The resulting <c>Group</c> object will manage the lifetime of the low-level MPI group,
/// and will call <see cref="Unsafe.MPI_Group_free"/> when it is disposed or finalized.
/// </summary>
/// <remarks>
/// This constructor should only be used in rare cases where the program
/// is manipulating MPI groups through the low-level MPI interface.
/// </remarks>
public static Group Adopt(MPI_Group group)
{
if (group == Unsafe.MPI_GROUP_NULL)
return null;
Group result = new Group();
result.group = group;
return result;
}
/// <summary>
/// Adopts a low-level MPI group that was created with any of the low-level MPI facilities.
/// The resulting <c>Group</c> object will manage the lifetime of the low-level MPI group,
/// and will call <see cref="Unsafe.MPI_Group_free"/> when it is disposed or finalized.
/// </summary>
/// <remarks>
/// This constructor should only be used in rare cases where the program
/// is manipulating MPI groups through the low-level MPI interface.
/// </remarks>
public static Group Adopt(MPI_Group group)
{
if (group == Unsafe.MPI_GROUP_NULL)
{
return(null);
}
Group result = new Group();
result.group = group;
return(result);
}
public static unsafe extern int MPI_Comm_remote_group(MPI_Comm comm, out MPI_Group group);
public static unsafe extern int MPI_Group_free(ref MPI_Group group);
public static unsafe extern int MPI_Comm_create(MPI_Comm comm, MPI_Group group, out MPI_Comm newcomm);
public static unsafe extern int MPI_Group_incl(MPI_Group group, int n, int[] ranks, out MPI_Group newgroup);
public static unsafe extern int MPI_Group_range_excl(MPI_Group group, int n, int[] ranges, out MPI_Group newgroup);
public static unsafe extern int MPI_Group_intersection(MPI_Group group1, MPI_Group group2, out MPI_Group newgroup);
public static unsafe extern int MPI_Group_difference(MPI_Group group1, MPI_Group group2, out MPI_Group newgroup);
public static unsafe extern int MPI_Group_compare(MPI_Group group1, MPI_Group group2, out int result);
public static unsafe extern int MPI_Group_translate_ranks (MPI_Group group1, int n, int[] ranks1, MPI_Group group2, int[] ranks2);
public static unsafe extern int MPI_Group_rank(MPI_Group group, out int rank);
public static unsafe extern int MPI_Group_size(MPI_Group group, out int size);
public void UpdateCoarseGrid()
{
// cout << "UpdateCoarseGrid" << endl;
// if (is_updated) return;
NgMPI_Comm comm = mesh.GetCommunicator();
int id = comm.Rank();
int ntasks = comm.Size();
if (ntasks == 1)
{
return;
}
Reset();
//C++ TO C# CONVERTER NOTE: This static local variable declaration (not allowed in C#) has been moved just prior to the method:
// static int timer = NgProfiler::CreateTimer("UpdateCoarseGrid");
NgProfiler.RegionTimer reg = new NgProfiler.RegionTimer(UpdateCoarseGrid_timer);
(*testout) << "UPDATE COARSE GRID PARALLEL TOPOLOGY " << "\n";
if (id == 0)
{
PrintMessage(1, "update parallel topology");
}
// UpdateCoarseGridGlobal();
// MPI_Barrier (MPI_COMM_WORLD);
MPI_Group MPI_GROUP_comm = new MPI_Group();
MPI_Group MPI_LocalGroup = new MPI_Group();
MPI_Comm MPI_LocalComm = new MPI_Comm();
int[] process_ranks = { 0 };
MPI_Comm_group(comm, MPI_GROUP_comm);
MPI_Group_excl(MPI_GROUP_comm, 1, process_ranks, MPI_LocalGroup);
MPI_Comm_create(comm, MPI_LocalGroup, MPI_LocalComm);
if (id == 0)
{
return;
}
MeshTopology topology = mesh.GetTopology();
Array <int> cnt_send = new Array <int>(ntasks - 1);
// update new vertices after mesh-refinement
if (mesh.mlbetweennodes.Size() > 0)
{
// cout << "UpdateCoarseGrid - vertices" << endl;
int newnv = mesh.mlbetweennodes.Size();
loc2distvert.ChangeSize(mesh.mlbetweennodes.Size());
/*
* for (PointIndex pi = PointIndex::BASE; pi < newnv+PointIndex::BASE; pi++)
* {
* PointIndex v1 = mesh.mlbetweennodes[pi][0];
* PointIndex v2 = mesh.mlbetweennodes[pi][1];
* if (mesh.mlbetweennodes[pi][0] != PointIndex::BASE-1)
* for (int dest = 1; dest < ntasks; dest++)
* if (IsExchangeVert (dest, v1) && IsExchangeVert (dest, v2))
* SetDistantPNum(dest, pi);
* }
*/
bool changed = true;
while (changed)
{
changed = false;
// build exchange vertices
cnt_send = 0;
foreach (PointIndex pi in mesh.Points().Range())
{
foreach (int dist in GetDistantPNums(pi - PointIndex.BASE))
{
cnt_send[dist - 1]++;
}
}
TABLE <int> dest2vert = new TABLE <int>(cnt_send);
foreach (PointIndex pi in mesh.Points().Range())
{
foreach (int dist in GetDistantPNums(pi - PointIndex.BASE))
{
dest2vert.Add(dist - 1, pi);
}
}
for (PointIndex pi = PointIndex.BASE; pi < newnv + PointIndex.BASE; pi++)
{
PointIndex v1 = mesh.mlbetweennodes[pi][0];
PointIndex v2 = mesh.mlbetweennodes[pi][1];
if (mesh.mlbetweennodes[pi][0] != PointIndex.BASE - 1)
{
// for (int dest = 1; dest < ntasks; dest++)
foreach (int dest in GetDistantPNums(v1 - PointIndex.BASE))
{
if (IsExchangeVert(dest, new netgen.PointIndex(v1)) && IsExchangeVert(dest, new netgen.PointIndex(v2)))
{
cnt_send[dest - 1]++;
}
}
}
}
TABLE <int> dest2pair = new TABLE <int>(cnt_send);
// for (int dest = 1; dest < ntasks; dest++)
for (PointIndex pi = PointIndex.BASE; pi < newnv + PointIndex.BASE; pi++)
{
PointIndex v1 = mesh.mlbetweennodes[pi][0];
PointIndex v2 = mesh.mlbetweennodes[pi][1];
if (mesh.mlbetweennodes[pi][0] != PointIndex.BASE - 1)
{
foreach (int dest in GetDistantPNums(v1 - PointIndex.BASE))
{
if (IsExchangeVert(dest, new netgen.PointIndex(v1)) && IsExchangeVert(dest, new netgen.PointIndex(v2)))
{
dest2pair.Add(dest - 1, pi);
}
}
}
}
cnt_send = 0;
int v1;
int v2;
for (PointIndex pi = PointIndex.BASE; pi < newnv + PointIndex.BASE; pi++)
{
PointIndex v1 = mesh.mlbetweennodes[pi][0];
PointIndex v2 = mesh.mlbetweennodes[pi][1];
if (mesh.mlbetweennodes[pi][0] != PointIndex.BASE - 1)
{
foreach (int dest in GetDistantPNums(v1 - PointIndex.BASE))
{
if (IsExchangeVert(dest, new netgen.PointIndex(v2)))
{
cnt_send[dest - 1] += 2;
}
}
}
}
TABLE <int> send_verts = new TABLE <int>(cnt_send);
Array <int, PointIndex.BASE> loc2exchange = new Array <int, PointIndex.BASE>(mesh.GetNV());
for (int dest = 1; dest < ntasks; dest++)
{
if (dest != id)
{
loc2exchange = -1;
int cnt = 0;
/*
* for (PointIndex pi : mesh.Points().Range())
* if (IsExchangeVert(dest, pi))
* loc2exchange[pi] = cnt++;
*/
foreach (PointIndex pi in dest2vert[dest - 1])
{
loc2exchange[pi] = cnt++;
}
// for (PointIndex pi = PointIndex::BASE; pi < newnv+PointIndex::BASE; pi++)
foreach (PointIndex pi in dest2pair[dest - 1])
{
PointIndex v1 = mesh.mlbetweennodes[pi][0];
PointIndex v2 = mesh.mlbetweennodes[pi][1];
if (mesh.mlbetweennodes[pi][0] != PointIndex.BASE - 1)
{
if (IsExchangeVert(dest, new netgen.PointIndex(v1)) && IsExchangeVert(dest, new netgen.PointIndex(v2)))
{
send_verts.Add(dest - 1, loc2exchange[v1]);
send_verts.Add(dest - 1, loc2exchange[v2]);
}
}
}
}
}
TABLE <int> recv_verts = new TABLE <int>(ntasks - 1);
netgen.GlobalMembers.MyMPI_ExchangeTable(send_verts, recv_verts, MPI_TAG_MESH + 9, MPI_LocalComm);
for (int dest = 1; dest < ntasks; dest++)
{
if (dest != id)
{
loc2exchange = -1;
int cnt = 0;
/*
* for (PointIndex pi : mesh.Points().Range())
* if (IsExchangeVert(dest, pi))
* loc2exchange[pi] = cnt++;
*/
foreach (PointIndex pi in dest2vert[dest - 1])
{
loc2exchange[pi] = cnt++;
}
FlatArray <int> recvarray = recv_verts[dest - 1];
for (int ii = 0; ii < recvarray.Size(); ii += 2)
{
foreach (PointIndex pi in dest2pair[dest - 1])
{
// for (PointIndex pi = PointIndex::BASE; pi < newnv+PointIndex::BASE; pi++)
PointIndex v1 = mesh.mlbetweennodes[pi][0];
PointIndex v2 = mesh.mlbetweennodes[pi][1];
if (mesh.mlbetweennodes[pi][0] != PointIndex.BASE - 1)
{
INDEX_2 re = new INDEX_2(recvarray[ii], recvarray[ii + 1]);
INDEX_2 es = new INDEX_2(loc2exchange[v1], loc2exchange[v2]);
if (es == re && !IsExchangeVert(dest, new netgen.PointIndex(pi)))
{
SetDistantPNum(dest, new netgen.PointIndex(pi));
changed = true;
}
}
}
}
}
}
}
}
Array <int> sendarray = new Array <int>();
Array <int> recvarray = new Array <int>();
// cout << "UpdateCoarseGrid - edges" << endl;
// static int timerv = NgProfiler::CreateTimer ("UpdateCoarseGrid - ex vertices");
//C++ TO C# CONVERTER NOTE: This static local variable declaration (not allowed in C#) has been moved just prior to the method:
// static int timere = NgProfiler::CreateTimer("UpdateCoarseGrid - ex edges");
//C++ TO C# CONVERTER NOTE: This static local variable declaration (not allowed in C#) has been moved just prior to the method:
// static int timerf = NgProfiler::CreateTimer("UpdateCoarseGrid - ex faces");
NgProfiler.StartTimer(UpdateCoarseGrid_timere);
int nfa = topology.GetNFaces();
int ned = topology.GetNEdges();
// build exchange vertices
cnt_send = 0;
foreach (PointIndex pi in mesh.Points().Range())
{
foreach (int dist in GetDistantPNums(pi - PointIndex.BASE))
{
cnt_send[dist - 1]++;
}
}
TABLE <int> dest2vert = new TABLE <int>(cnt_send);
foreach (PointIndex pi in mesh.Points().Range())
{
foreach (int dist in GetDistantPNums(pi - PointIndex.BASE))
{
dest2vert.Add(dist - 1, pi);
}
}
// exchange edges
cnt_send = 0;
int v1;
int v2;
for (int edge = 1; edge <= ned; edge++)
{
topology.GetEdgeVertices(edge, ref v1, ref v2);
for (int dest = 1; dest < ntasks; dest++)
{
if (IsExchangeVert(dest, v1) && IsExchangeVert(dest, v2))
{
cnt_send[dest - 1] += 1;
}
}
}
TABLE <int> dest2edge = new TABLE <int>(cnt_send);
foreach (int v in cnt_send)
{
v *= 2;
}
TABLE <int> send_edges = new TABLE <int>(cnt_send);
for (int edge = 1; edge <= ned; edge++)
{
topology.GetEdgeVertices(edge, ref v1, ref v2);
for (int dest = 1; dest < ntasks; dest++)
{
if (IsExchangeVert(dest, v1) && IsExchangeVert(dest, v2))
{
dest2edge.Add(dest - 1, edge);
}
}
}
Array <int, PointIndex.BASE> loc2exchange = new Array <int, PointIndex.BASE>(mesh.GetNV());
for (int dest = 1; dest < ntasks; dest++)
{
loc2exchange = -1;
int cnt = 0;
foreach (PointIndex pi in dest2vert[dest - 1])
{
loc2exchange[pi] = cnt++;
}
foreach (int edge in dest2edge[dest - 1])
{
topology.GetEdgeVertices(edge, ref v1, ref v2);
if (IsExchangeVert(dest, v1) && IsExchangeVert(dest, v2))
{
send_edges.Add(dest - 1, loc2exchange[v1]);
send_edges.Add(dest - 1, loc2exchange[v2]);
}
}
}
// cout << "UpdateCoarseGrid - edges mpi-exchange" << endl;
TABLE <int> recv_edges = new TABLE <int>(ntasks - 1);
netgen.GlobalMembers.MyMPI_ExchangeTable(send_edges, recv_edges, MPI_TAG_MESH + 9, MPI_LocalComm);
// cout << "UpdateCoarseGrid - edges mpi-exchange done" << endl;
/*
* for (int dest = 1; dest < ntasks; dest++)
* {
* auto ex2loc = dest2vert[dest-1];
* FlatArray<int> recvarray = recv_edges[dest-1];
* for (int ii = 0; ii < recvarray.Size(); ii+=2)
* for (int edge : dest2edge[dest-1])
* {
* topology.GetEdgeVertices (edge, v1, v2);
* INDEX_2 re(ex2loc[recvarray[ii]],
* ex2loc[recvarray[ii+1]]);
* INDEX_2 es(v1, v2);
* if (es == re)
* SetDistantEdgeNum(dest, edge);
* }
* }
*/
for (int dest = 1; dest < ntasks; dest++)
{
var ex2loc = dest2vert[dest - 1];
if (ex2loc.Size() == 0)
{
continue;
}
INDEX_2_CLOSED_HASHTABLE <int> vert2edge = new INDEX_2_CLOSED_HASHTABLE <int>((uint)(2 * dest2edge[dest - 1].Size() + 10));
foreach (int edge in dest2edge[dest - 1])
{
topology.GetEdgeVertices(edge, ref v1, ref v2);
vert2edge.Set(new INDEX_2(v1, v2), edge);
}
FlatArray <int> recvarray = recv_edges[dest - 1];
for (int ii = 0; ii < recvarray.Size(); ii += 2)
{
INDEX_2 re = new INDEX_2(ex2loc[recvarray[ii]], ex2loc[recvarray[ii + 1]]);
if (vert2edge.Used(re))
{
SetDistantEdgeNum(dest, vert2edge.Get(re));
}
}
}
NgProfiler.StopTimer(UpdateCoarseGrid_timere);
// MPI_Barrier (MPI_LocalComm);
// cout << "UpdateCoarseGrid - faces" << endl;
if (mesh.GetDimension() == 3)
{
NgProfiler.StartTimer(UpdateCoarseGrid_timerf);
Array <int> verts = new Array <int>();
// exchange faces
cnt_send = 0;
for (int face = 1; face <= nfa; face++)
{
topology.GetFaceVertices(face, verts);
for (int dest = 1; dest < ntasks; dest++)
{
if (dest != id)
{
if (IsExchangeVert(dest, verts[0]) && IsExchangeVert(dest, verts[1]) && IsExchangeVert(dest, verts[2]))
{
cnt_send[dest - 1]++;
}
}
}
}
TABLE <int> dest2face = new TABLE <int>(cnt_send);
for (int face = 1; face <= nfa; face++)
{
topology.GetFaceVertices(face, verts);
for (int dest = 1; dest < ntasks; dest++)
{
if (dest != id)
{
if (IsExchangeVert(dest, verts[0]) && IsExchangeVert(dest, verts[1]) && IsExchangeVert(dest, verts[2]))
{
dest2face.Add(dest - 1, face);
}
}
}
}
foreach (int c in cnt_send)
{
c *= 3;
}
TABLE <int> send_faces = new TABLE <int>(cnt_send);
Array <int, PointIndex.BASE> loc2exchange = new Array <int, PointIndex.BASE>(mesh.GetNV());
for (int dest = 1; dest < ntasks; dest++)
{
if (dest != id)
{
/*
* loc2exchange = -1;
* int cnt = 0;
* for (PointIndex pi : mesh.Points().Range())
* if (IsExchangeVert(dest, pi))
* loc2exchange[pi] = cnt++;
*/
if (dest2vert[dest - 1].Size() == 0)
{
continue;
}
loc2exchange = -1;
int cnt = 0;
foreach (PointIndex pi in dest2vert[dest - 1])
{
loc2exchange[pi] = cnt++;
}
foreach (int face in dest2face[dest - 1])
{
topology.GetFaceVertices(face, verts);
if (IsExchangeVert(dest, verts[0]) && IsExchangeVert(dest, verts[1]) && IsExchangeVert(dest, verts[2]))
{
send_faces.Add(dest - 1, loc2exchange[verts[0]]);
send_faces.Add(dest - 1, loc2exchange[verts[1]]);
send_faces.Add(dest - 1, loc2exchange[verts[2]]);
}
}
}
}
// cout << "UpdateCoarseGrid - faces mpi-exchange" << endl;
TABLE <int> recv_faces = new TABLE <int>(ntasks - 1);
netgen.GlobalMembers.MyMPI_ExchangeTable(send_faces, recv_faces, MPI_TAG_MESH + 9, MPI_LocalComm);
// cout << "UpdateCoarseGrid - faces mpi-exchange done" << endl;
/*
* for (int dest = 1; dest < ntasks; dest++)
* if (dest != id)
* {
* loc2exchange = -1;
* int cnt = 0;
* for (PointIndex pi : dest2vert[dest-1])
* loc2exchange[pi] = cnt++;
*
* FlatArray<int> recvarray = recv_faces[dest-1];
* for (int ii = 0; ii < recvarray.Size(); ii+=3)
* for (int face : dest2face[dest-1])
* {
* topology.GetFaceVertices (face, verts);
* INDEX_3 re(recvarray[ii], recvarray[ii+1], recvarray[ii+2]);
* INDEX_3 es(loc2exchange[verts[0]], loc2exchange[verts[1]], loc2exchange[verts[2]]);
* if (es == re)
* SetDistantFaceNum(dest, face);
* }
* }
*/
for (int dest = 1; dest < ntasks; dest++)
{
var ex2loc = dest2vert[dest - 1];
if (ex2loc.Size() == 0)
{
continue;
}
INDEX_3_CLOSED_HASHTABLE <int> vert2face = new INDEX_3_CLOSED_HASHTABLE <int>(2 * dest2face[dest - 1].Size() + 10);
foreach (int face in dest2face[dest - 1])
{
topology.GetFaceVertices(face, verts);
vert2face.Set(new INDEX_3(verts[0], verts[1], verts[2]), face);
}
FlatArray <int> recvarray = recv_faces[dest - 1];
for (int ii = 0; ii < recvarray.Size(); ii += 3)
{
INDEX_3 re = new INDEX_3(ex2loc[recvarray[ii]], ex2loc[recvarray[ii + 1]], ex2loc[recvarray[ii + 2]]);
if (vert2face.Used(re))
{
SetDistantFaceNum(dest, vert2face.Get(re));
}
}
}
/*
* Array<int,1> glob2loc;
*
* int maxface = 0;
* for (int face = 1; face <= nfa; face++)
* maxface = max (maxface, GetGlobalFaceNum (face));
*
* // glob2loc.SetSize (nfaglob);
* glob2loc.SetSize (maxface);
* glob2loc = -1;
*
* for (int loc = 1; loc <= nfa; loc++)
* glob2loc[GetGlobalFaceNum(loc)] = loc;
*
* cnt_send = 0;
* Array<int> verts;
* for (int face = 1; face <= nfa; face++)
* {
* topology.GetFaceVertices (face, verts);
* for (int dest = 1; dest < ntasks; dest++)
* if (IsExchangeVert (dest, verts[0]) &&
* IsExchangeVert (dest, verts[1]) &&
* IsExchangeVert (dest, verts[2]))
* {
* cnt_send[dest-1]+=2;
* }
* }
*
* TABLE<int> send_faces(cnt_send);
* for (int face = 1; face <= nfa; face++)
* {
* topology.GetFaceVertices (face, verts);
* for (int dest = 1; dest < ntasks; dest++)
* {
* if (IsExchangeVert (dest, verts[0]) &&
* IsExchangeVert (dest, verts[1]) &&
* IsExchangeVert (dest, verts[2]))
* {
* send_faces.Add (dest-1, GetGlobalFaceNum(face));
* send_faces.Add (dest-1, face);
* }
* }
* }
* TABLE<int> recv_faces(ntasks-1);
* MyMPI_ExchangeTable (send_faces, recv_faces, MPI_TAG_MESH+8, MPI_LocalComm);
*
* for (int sender = 1; sender < ntasks; sender ++)
* if (id != sender)
* {
* FlatArray<int> recvarray = recv_faces[sender-1];
*
* for (int ii = 0; ii < recvarray.Size(); )
* {
* int globf = recvarray[ii++];
* int distf = recvarray[ii++];
*
* if (globf <= maxface)
* {
* int locf = glob2loc[globf];
* if (locf != -1)
* SetDistantFaceNum (sender, locf);
* }
* }
* }
*/
NgProfiler.StopTimer(UpdateCoarseGrid_timerf);
}
// cout << "UpdateCoarseGrid - done" << endl;
is_updated = true;
}
