/// <summary>
///
/// </summary>
/// <param name="TargetMappingIndex"></param>
/// <param name="outputPartitioning">
/// Partitioning of the new grid, resp the return array.
/// </param>
/// <returns>
/// - 1st index: cell index in new grid, correlates with <paramref name="outputPartitioning"/>.
/// - 2nd index: enumeration over cells (in the old grid) which are combined in the new grid.
/// For cells with refinement, always one entry, for cells which are coarsened a greater number of entries.
/// If null, the cell is not changed.
/// - content: Subdivision leaf index, correlates with 2nd index of <see cref="KrefS_SubdivLeaves"/>, can be used as an input to <see cref="GetSubdivBasisTransform(int, int, int)"/>.
/// </returns>
public int[][] GetTargetMappingIndex(IPartitioning outputPartitioning)
{
using (new FuncTrace()) {
Debug.Assert(DestGlobalId.Length == MappingIndex.Length);
Debug.Assert(OldGlobalId.Length == MappingIndex.Length);
int oldJ = DestGlobalId.Length;
// Caching
// =======
if (m_TargetMappingIndex != null)
{
// caching
if (m_TargetMappingIndex.Length != outputPartitioning.LocalLength)
{
throw new ArgumentException("Length mismatch of output list and output partition.");
}
return(m_TargetMappingIndex);
}
// local evaluation, prepare communication
// =======================================
m_TargetMappingIndex = new int[outputPartitioning.LocalLength][];
int j0Dest = outputPartitioning.i0;
// keys: processors which should receive data from this processor
Dictionary <int, GetTargetMapping_Helper> AllSendData = new Dictionary <int, GetTargetMapping_Helper>();
for (int j = 0; j < oldJ; j++)
{
int[] MappingIndex_j = MappingIndex[j];
if (MappingIndex_j != null)
{
Debug.Assert(TargetIdx[j].Length == MappingIndex_j.Length);
int L = MappingIndex_j.Length;
for (int l = 0; l < L; l++)
{
int jDest = TargetIdx[j][l];
int MapIdx = MappingIndex_j[l];
if (outputPartitioning.IsInLocalRange(jDest))
{
int[] destCollection = m_TargetMappingIndex[jDest - j0Dest];
ArrayTools.AddToArray(MapIdx, ref destCollection);
m_TargetMappingIndex[jDest - j0Dest] = destCollection;
}
else
{
int targProc = outputPartitioning.FindProcess(jDest);
GetTargetMapping_Helper dataTargPrc;
if (!AllSendData.TryGetValue(targProc, out dataTargPrc))
{
dataTargPrc = new GetTargetMapping_Helper();
AllSendData.Add(targProc, dataTargPrc);
}
dataTargPrc.TargetIndices.Add(jDest);
dataTargPrc.Items.Add(MapIdx);
}
}
}
else
{
Debug.Assert(TargetIdx[j].Length == 1);
}
}
// communication
// =============
var AllRcvData = SerialisationMessenger.ExchangeData(AllSendData, outputPartitioning.MPI_Comm);
foreach (var kv in AllRcvData)
{
int rcvProc = kv.Key;
j0Dest = outputPartitioning.GetI0Offest(rcvProc);
var TIdxs = kv.Value.TargetIndices;
var TVals = kv.Value.Items;
Debug.Assert(TIdxs.Count == TVals.Count);
int L = TIdxs.Count;
for (int l = 0; l < L; l++)
{
int idx = TIdxs[l] - j0Dest;
Debug.Assert(outputPartitioning.IsInLocalRange(idx));
int[] destCollection = m_TargetMappingIndex[idx];
ArrayTools.AddToArray(TVals[idx], ref destCollection);
m_TargetMappingIndex[idx] = destCollection;
}
}
// return
// ======
return(m_TargetMappingIndex);
}
}
private static byte[] SyncEdgeTagsOverMPI(Dictionary <string, byte> EdgeTagNames_Reverse)
{
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out int MyRank);
var To0 = new Dictionary <int, KeyValuePair <string, byte>[]>();
if (MyRank > 0)
{
To0.Add(0, EdgeTagNames_Reverse.ToArray());
}
var allData = SerialisationMessenger.ExchangeData(To0);
bool[] usedEdgeTags = new bool[byte.MaxValue + 1];
foreach (var et in EdgeTagNames_Reverse.Values)
{
usedEdgeTags[et] = true;
}
byte GetNewEt()
{
for (int i = 1; i < usedEdgeTags.Length; i++)
{
if (i >= GridCommons.FIRST_PERIODIC_BC_TAG)
{
throw new ApplicationException("Running out of edge tags.");
}
if (usedEdgeTags[i] == false)
{
usedEdgeTags[i] = true;
return((byte)i);
}
}
throw new ApplicationException("Running out of edge tags.");
}
if (MyRank == 0)
{
foreach (var kv in allData)
{
var backData = kv.Value;
for (int i = 0; i < backData.Length; i++)
{
if (EdgeTagNames_Reverse.ContainsKey(backData[i].Key))
{
}
else
{
byte sugKey = backData[i].Value;
if (usedEdgeTags[sugKey])
{
sugKey = GetNewEt();
}
EdgeTagNames_Reverse.Add(backData[i].Key, sugKey);
}
}
}
}
var AllEts = EdgeTagNames_Reverse.ToArray().MPIBroadcast(0);
byte[] EtTanslations = (byte.MaxValue + 1).ForLoop(i => (byte)i);
bool AnyTranslation = false;
if (MyRank > 0)
{
foreach (var kv in AllEts)
{
if (EdgeTagNames_Reverse.ContainsKey(kv.Key))
{
byte oldVal = EdgeTagNames_Reverse[kv.Key];
byte newVal = kv.Value;
AnyTranslation = AnyTranslation | (oldVal != newVal);
EdgeTagNames_Reverse[kv.Key] = newVal;
EtTanslations[oldVal] = newVal;
}
}
}
AnyTranslation = AnyTranslation.MPIOr();
if (AnyTranslation)
{
return(EtTanslations);
}
else
{
return(null);
}
}
public void ApplyToVector <I>(IList <I> input, IList <I[]> output, IPartitioning outputPartitioning)
{
using (new FuncTrace()) {
Debug.Assert(DestGlobalId.Length == MappingIndex.Length);
Debug.Assert(OldGlobalId.Length == MappingIndex.Length);
int oldJ = DestGlobalId.Length;
if (input.Count != oldJ)
{
throw new ArgumentException("Mismatch between input vector length and current data length.");
}
if (output.Count != outputPartitioning.LocalLength)
{
throw new ArgumentException("Length mismatch of output list and output partition.");
}
int j0Dest = outputPartitioning.i0;
// keys: processors which should receive data from this processor
Dictionary <int, ApplyToVector_Helper <I> > AllSendData = new Dictionary <int, ApplyToVector_Helper <I> >();
for (int j = 0; j < oldJ; j++)
{
I data_j = input[j];
foreach (int jDest in TargetIdx[j])
{
if (outputPartitioning.IsInLocalRange(jDest))
{
I[] destCollection = output[jDest - j0Dest];
ArrayTools.AddToArray(data_j, ref destCollection);
output[jDest - j0Dest] = destCollection;
}
else
{
int targProc = outputPartitioning.FindProcess(jDest);
ApplyToVector_Helper <I> dataTargPrc;
if (!AllSendData.TryGetValue(targProc, out dataTargPrc))
{
dataTargPrc = new ApplyToVector_Helper <I>();
AllSendData.Add(targProc, dataTargPrc);
}
dataTargPrc.TargetIndices.Add(jDest);
dataTargPrc.Items.Add(data_j);
}
}
}
var AllRcvData = SerialisationMessenger.ExchangeData(AllSendData, outputPartitioning.MPI_Comm);
foreach (var kv in AllRcvData)
{
int rcvProc = kv.Key;
j0Dest = outputPartitioning.GetI0Offest(rcvProc);
var TIdxs = kv.Value.TargetIndices;
var TVals = kv.Value.Items;
Debug.Assert(TIdxs.Count == TVals.Count);
int L = TIdxs.Count;
for (int l = 0; l < L; l++)
{
int idx = TIdxs[l] - j0Dest;
Debug.Assert(outputPartitioning.IsInLocalRange(idx));
I[] destCollection = output[idx];
ArrayTools.AddToArray(TVals[idx], ref destCollection);
output[idx] = destCollection;
}
}
}
}
/// <summary>
/// Like <see cref="Evaluate(double, IEnumerable{DGField}, MultidimensionalArray, double, MultidimensionalArray, BitArray, int[])"/>,
/// but with MPI-Exchange
/// </summary>
public int EvaluateParallel(double alpha, IEnumerable <DGField> Flds, MultidimensionalArray Points, double beta, MultidimensionalArray Result, BitArray UnlocatedPoints = null)
{
using (new FuncTrace()) {
MPICollectiveWatchDog.Watch();
int L = Points != null ? Points.NoOfRows : 0;
int MPIsize = m_Context.MpiSize;
int D = m_Context.SpatialDimension;
if (UnlocatedPoints != null)
{
if (UnlocatedPoints.Length != L)
{
throw new ArgumentException("Length mismatch");
}
}
// evaluate locally
// ================
var unlocated = new System.Collections.BitArray(L);
int NoOfUnlocated;
if (L > 0)
{
NoOfUnlocated = this.Evaluate(alpha, Flds, Points, beta, Result, unlocated);
}
else
{
NoOfUnlocated = 0;
}
// return, if there are no unlocalized points
// ==========================================
int TotNoOfUnlocated = NoOfUnlocated.MPISum();
if (TotNoOfUnlocated <= 0)
{
if (UnlocatedPoints != null)
{
UnlocatedPoints.SetAll(false);
}
return(0);
}
// copy unlocalized to separate array
// ==================================
double[,] localUnlocated = new double[NoOfUnlocated, D]; // MultidimensionalArray does not allow zero length -- so use double[,] instead
int[] IndexToOrgIndex = new int[NoOfUnlocated];
int u = 0;
for (int i = 0; i < L; i++)
{
if (unlocated[i])
{
localUnlocated.SetRowPt(u, Points.GetRowPt(i));
IndexToOrgIndex[u] = i;
u++;
}
}
Debug.Assert(u == NoOfUnlocated);
// collect on all ranks -- this won't scale well, but it may work
// ==============================================================
MultidimensionalArray globalUnlocated;
int[] WhoIsInterestedIn; // index: point index, corresponds with 'globalUnlocated' rows; content: rank which needs the result
int[] OriginalIndex; // index: detto; content: index which the point had on the processor that sent it.
double[][,] __globalUnlocated;
int LL;
{
__globalUnlocated = localUnlocated.MPIAllGatherO();
Debug.Assert(__globalUnlocated.Length == MPIsize);
Debug.Assert(__globalUnlocated.Select(aa => aa.GetLength(0)).Sum() == TotNoOfUnlocated);
Debug.Assert(__globalUnlocated[m_Context.MpiRank].GetLength(0) == NoOfUnlocated);
LL = TotNoOfUnlocated - NoOfUnlocated;
if (LL > 0)
{
globalUnlocated = MultidimensionalArray.Create(LL, D);
}
else
{
globalUnlocated = null;
}
WhoIsInterestedIn = new int[LL];
OriginalIndex = new int[LL];
int g = 0;
for (int r = 0; r < MPIsize; r++) // concat all point arrays from all processors
{
if (r == m_Context.MpiRank)
{
continue;
}
double[,] __globalPart = __globalUnlocated[r];
int Lr = __globalPart.GetLength(0);
if (Lr > 0)
{
globalUnlocated.ExtractSubArrayShallow(new[] { g, 0 }, new[] { g + Lr - 1, D - 1 }).Acc2DArray(1.0, __globalPart);
}
for (int i = 0; i < Lr; i++)
{
WhoIsInterestedIn[i + g] = r;
OriginalIndex[i + g] = i;
}
g += Lr;
}
}
// try to evaluate the so-far-unlocalized points
// ---------------------------------------------
var unlocated2 = new System.Collections.BitArray(LL);
var Result2 = LL > 0 ? MultidimensionalArray.Create(LL, Flds.Count()) : null;
int NoOfUnlocated2 = LL > 0 ? this.Evaluate(1.0, Flds, globalUnlocated, 0.0, Result2, unlocated2) : 0;
// backward MPI sending
// --------------------
IDictionary <int, EvaluateParallelHelper> resultFromOtherProcs;
{
var backSend = new Dictionary <int, EvaluateParallelHelper>();
for (int ll = 0; ll < LL; ll++)
{
if (!unlocated2[ll])
{
int iTarget = WhoIsInterestedIn[ll];
Debug.Assert(iTarget != m_Context.MpiRank);
if (!backSend.TryGetValue(iTarget, out EvaluateParallelHelper eph))
{
eph = new EvaluateParallelHelper();
backSend.Add(iTarget, eph);
}
eph.OriginalIndices.Add(OriginalIndex[ll]);
eph.Results.Add(Result2.GetRow(ll));
}
}
resultFromOtherProcs = SerialisationMessenger.ExchangeData(backSend);
}
// fill the results from other processors
// ======================================
foreach (var res in resultFromOtherProcs.Values)
{
int K = res.OriginalIndices.Count();
Debug.Assert(res.OriginalIndices.Count == res.Results.Count);
for (int k = 0; k < K; k++)
{
int iOrg = IndexToOrgIndex[res.OriginalIndices[k]];
if (unlocated[iOrg] == true)
{
NoOfUnlocated--;
}
unlocated[iOrg] = false;
Result.AccRow(iOrg, alpha, res.Results[k]);
}
}
// Return
// ======
if (UnlocatedPoints != null)
{
for (int l = 0; l < L; l++)
{
UnlocatedPoints[l] = unlocated[l];
}
}
return(NoOfUnlocated);
}
}
/// <summary>
/// Computes the neighbor cells globally (i.e. over all MPI processors) for each local cell.
/// </summary>
/// <returns>
/// <param name="IncludeBcCells">
/// If true, also the boundary condition cells (<see cref="BcCells"/>) will be included in the output array.
/// </param>
/// Cell-wise neighborship information:
/// - index: local cell index <em>j</em>, i.e. correlates with <see cref="Cells"/>; if <paramref name="IncludeBcCells"/> is true,
/// the information for boundary cells is added after the information for cells.
/// - content: for the index <em>j</em> the set of neighbor cells. If the global index (<see cref="Neighbour.Neighbour_GlobalIndex"/>)
/// is greater or equal than the global number of cells (<see cref="NumberOfCells"/>) the neighbor is a boundary condition cell,
/// (<see cref="BcCells"/>).
/// </returns>
public IEnumerable <Neighbour>[] GetCellNeighbourship(bool IncludeBcCells)
{
ilPSP.MPICollectiveWatchDog.Watch();
using (new FuncTrace()) {
var ftNeigh = GetFaceTagsNeigbourIndices(IncludeBcCells);
var NPart = this.NodePartitioning;
int K = NPart.LocalLength;
int k0 = NPart.i0;
int J = this.NoOfUpdateCells;
int J_BC = IncludeBcCells ? this.NoOfBcCells : 0;
int j0 = this.CellPartitioning.i0;
int Jglob = this.CellPartitioning.TotalLength;
int j0Bc = this.BcCellPartitioning.i0;
// Which cells make use of a particular node?
//-------------------------------------------
// Index: Node index
// Entry: Enumeration of global indices of cells that use this
// particular node
List <int>[] Nodes2Cells = new List <int> [K];
{
for (int k = 0; k < K; k++)
{
Nodes2Cells[k] = new List <int>();
}
// key: MPI processor rank
// value: information packet
Dictionary <int, List <NodeCellIndexPair> > Y =
new Dictionary <int, List <NodeCellIndexPair> >();
for (int j = 0; j < (J + J_BC); j++)
{
Element Cell_j;
RefElement Kref;
int jCell_glob;
if (j < J)
{
Cell_j = this.Cells[j];
Kref = this.m_RefElements.Single(KK => KK.SupportedCellTypes.Contains(Cell_j.Type));
jCell_glob = j + j0;
}
else
{
Cell_j = this.BcCells[j - J];
Kref = this.m_EdgeRefElements.Single(KK => KK.SupportedCellTypes.Contains(Cell_j.Type));
jCell_glob = (j - J) + j0Bc + Jglob;
}
var CellNodes = Cell_j.NodeIndices;
if (CellNodes.Length != Kref.NoOfVertices)
{
throw new ApplicationException("error in data structure.");
}
foreach (int NodeId in CellNodes)
{
int target_prozi = NPart.FindProcess(NodeId);
if (target_prozi == MyRank)
{
Nodes2Cells[NodeId - k0].Add(jCell_glob);
}
else
{
NodeCellIndexPair Packet;
Packet.NodeId = NodeId;
Packet.GlobalCellIndex = jCell_glob;
List <NodeCellIndexPair> Z;
if (!Y.TryGetValue(target_prozi, out Z))
{
Z = new List <NodeCellIndexPair>();
Y.Add(target_prozi, Z);
}
Z.Add(Packet);
}
}
}
var W = SerialisationMessenger.ExchangeData(Y, csMPI.Raw._COMM.WORLD);
foreach (var wp in W.Values)
{
foreach (NodeCellIndexPair Packet in wp)
{
Nodes2Cells[Packet.NodeId - k0].Add(Packet.GlobalCellIndex);
}
}
}
// For every cell, for every vertex in this cell:
// Which other cells die also use this node?
//-----------------------------------------------
// 1st index: Local cell index
// 2nd index: Cell vertex index
// 3rd index: Collection of 'peer' cells
int[][][] NodePeers = new int[J + J_BC][][];
{
for (int j = 0; j < J + J_BC; j++)
{
Element Cell_j;
if (j < J)
{
Cell_j = this.Cells[j];
}
else
{
Cell_j = this.BcCells[j - J];
}
NodePeers[j] = new int[Cell_j.NodeIndices.Length][];
}
Dictionary <int, List <NodeCellListPair> > Y = new Dictionary <int, List <NodeCellListPair> >();
var CPart = this.CellPartitioning;
var BcPart = this.BcCellPartitioning;
for (int k = 0; k < K; k++) // loop over locally assigned nodes
{
int k_node = k + k0;
var cell_list = Nodes2Cells[k].ToArray();
foreach (int jCell in cell_list) // loop over all cells that use node 'k'
{
int cell_proc;
int local_offset;
if (jCell < Jglob)
{
// normal cell
cell_proc = CPart.FindProcess(jCell);
local_offset = j0;
}
else
{
// boundary condition cell
cell_proc = BcPart.FindProcess(jCell - Jglob);
local_offset = Jglob + j0Bc;
}
if (cell_proc == MyRank)
{
int jCell_loc = jCell - local_offset;
int kC;
bool bfound = false;
Element Cell_j;
int oo;
if (jCell < Jglob)
{
// normal cell
Cell_j = this.Cells[jCell_loc];
oo = 0;
}
else
{
// boundary condition cell
Cell_j = this.BcCells[jCell_loc];
oo = J;
}
for (kC = 0; kC < Cell_j.NodeIndices.Length; kC++)
{
if (Cell_j.NodeIndices[kC] == k_node)
{
bfound = true;
break;
}
}
if (!bfound)
{
throw new ApplicationException("error in algorithm.");
}
NodePeers[jCell_loc + oo][kC] = cell_list;
}
else
{
NodeCellListPair A;
A.NodeId = k_node;
A.CellList = cell_list;
List <NodeCellListPair> Z;
if (!Y.TryGetValue(cell_proc, out Z))
{
Z = new List <NodeCellListPair>();
Y.Add(cell_proc, Z);
}
Z.Add(A);
}
}
}
var W = SerialisationMessenger.ExchangeData(Y, csMPI.Raw._COMM.WORLD);
foreach (var wp in W.Values)
{
foreach (var P in wp)
{
int k_node = P.NodeId;
int[] cell_list = P.CellList;
foreach (int jCell in cell_list)
{
int cell_proc;
int local_offset;
if (jCell < Jglob)
{
// normal cell
cell_proc = CPart.FindProcess(jCell);
local_offset = j0;
}
else
{
// boundary condition cell
cell_proc = BcPart.FindProcess(jCell - Jglob);
local_offset = Jglob + j0Bc;
}
if (cell_proc == MyRank)
{
int jCell_loc = jCell - local_offset;
Element Cell_j;
int oo;
if (jCell < Jglob)
{
// normal cell
Cell_j = this.Cells[jCell_loc];
oo = 0;
}
else
{
// boundary condition cell
Cell_j = this.BcCells[jCell_loc];
oo = J;
}
int kC;
bool bfound = false;
for (kC = 0; kC < Cell_j.NodeIndices.Length; kC++)
{
if (Cell_j.NodeIndices[kC] == k_node)
{
bfound = true;
break;
}
}
if (!bfound)
{
throw new ApplicationException("error in algorithm.");
}
NodePeers[jCell_loc + oo][kC] = cell_list;
}
}
}
}
}
// Assemble final result
// ---------------------
IEnumerable <Neighbour>[] CellNeighbours;
{
CellNeighbours = new IEnumerable <Neighbour> [J + J_BC];
for (int j = 0; j < J + J_BC; j++) // loop over cells
//var Cell_j = this.Cells[j];
//int jCellGlob = j + j0;
//var Kref = this.m_GridSimplices.Single(KK => KK.SupportedTypes.Contains(Cell_j.Type));
{
Element Cell_j;
RefElement Kref;
int jCellGlob;
if (j < J)
{
Cell_j = this.Cells[j];
Kref = this.m_RefElements.Single(KK => KK.SupportedCellTypes.Contains(Cell_j.Type));
jCellGlob = j + j0;
}
else
{
Cell_j = this.BcCells[j - J];
Kref = this.m_EdgeRefElements.Single(KK => KK.SupportedCellTypes.Contains(Cell_j.Type));
jCellGlob = (j - J) + j0Bc + Jglob;
}
var Cell_j_Neighs = new List <Neighbour>();
CellNeighbours[j] = Cell_j_Neighs;
// find neighbor cells connected via grid nodes
// --------------------------------------------
if (j < J)
{
//normal cells: match faces
var faceVtx = Kref.FaceToVertexIndices;
int[][] B = new int[faceVtx.GetLength(1)][];
for (int _iface = 0; _iface < Kref.NoOfFaces; _iface++) // loop over faces of cell 'j' (local index) resp. 'jCellGlob' (global index)
{
for (int iv = 0; iv < B.Length; iv++)
{
B[iv] = NodePeers[j][faceVtx[_iface, iv]];
}
int NeighIdx = Intersect(B, jCellGlob);
if (NeighIdx >= 0)
{
Neighbour nCN = default(Neighbour);
nCN.Neighbour_GlobalIndex = NeighIdx;
nCN.CellFaceTag.FaceIndex = _iface;
nCN.CellFaceTag.ConformalNeighborship = true;
Cell_j_Neighs.Add(nCN);
}
}
}
else
{
// boundary-condition cell: match the whole element
int[][] B = new int[Kref.NoOfVertices][];
for (int iv = 0; iv < B.Length; iv++)
{
B[iv] = NodePeers[j][iv];
}
int NeighIdx = Intersect(B, jCellGlob);
if (NeighIdx >= 0)
{
Neighbour nCN = default(Neighbour);
nCN.Neighbour_GlobalIndex = NeighIdx;
nCN.CellFaceTag.FaceIndex = -1;
nCN.CellFaceTag.ConformalNeighborship = true;
Cell_j_Neighs.Add(nCN);
}
}
// find neighbor cells connected via CellFaceTag's
// -----------------------------------------------
var otherNeighbours = ftNeigh[j]; // ftNeigh is the result of CellFaceTag-based connectivity
if (j < J)
{
var _Cell_j = (Cell)Cell_j;
Debug.Assert(((otherNeighbours == null ? 0 : otherNeighbours.Length) == ((_Cell_j.CellFaceTags == null) ? 0 : _Cell_j.CellFaceTags.Length)));
if (otherNeighbours != null)
{
for (int w = 0; w < otherNeighbours.Length; w++)
{
Debug.Assert(_Cell_j.CellFaceTags[w].NeighCell_GlobalID < 0 == otherNeighbours[w] < 0);
if (_Cell_j.CellFaceTags[w].NeighCell_GlobalID >= 0)
{
if (Cell_j_Neighs.Where(neigh => neigh.Neighbour_GlobalIndex == otherNeighbours[w]).Count() <= 0) // filter duplicates
{
Cell_j_Neighs.Add(new Neighbour()
{
Neighbour_GlobalIndex = otherNeighbours[w],
CellFaceTag = _Cell_j.CellFaceTags[w],
});
}
}
}
}
}
else
{
var BcCell_j = (BCElement)Cell_j;
Debug.Assert(((otherNeighbours == null ? 0 : otherNeighbours.Length) == ((BcCell_j.NeighCell_GlobalIDs == null) ? 0 : BcCell_j.NeighCell_GlobalIDs.Length)));
if (otherNeighbours != null)
{
for (int w = 0; w < otherNeighbours.Length; w++)
{
Cell_j_Neighs.Add(new Neighbour()
{
Neighbour_GlobalIndex = otherNeighbours[w],
CellFaceTag = new CellFaceTag()
{
EdgeTag = BcCell_j.EdgeTag,
FaceIndex = int.MinValue,
NeighCell_GlobalID = BcCell_j.NeighCell_GlobalIDs[w],
ConformalNeighborship = BcCell_j.Conformal
}
});
}
}
}
}
}
return(CellNeighbours);
}
}
/// <summary>
/// ctor.
/// </summary>
/// <param name="M"></param>
/// <param name="ExtCol">
/// key: processor rank 'p' <br/>
/// value: a list of column indices (within the local range of columns of <paramref name="M"/>),
/// which should be editable at rank 'p'.
/// </param>
public MsrExtMatrix(IMutableMatrixEx M, IDictionary <int, int[]> ExtCol)
{
this.ColPart = M.ColPartition;
int i0Row = (int)M.RowPartitioning.i0, I = M.RowPartitioning.LocalLength,
i0Col = (int)this.ColPart.i0, J = this.ColPart.LocalLength;
Mtx = M;
// init
// ====
ColToRowLocal = new List <int> [ColPart.LocalLength];
for (int j = 0; j < ColToRowLocal.Length; j++)
{
ColToRowLocal[j] = new List <int>();
}
// build Column to row - mapping
// =============================
ColToRowExternal = new Dictionary <int, List <int> >();
// key: global column index j, within the range of processor 'p'
// values: global row indices
SortedDictionary <int, List <int> > ColForProc = new SortedDictionary <int, List <int> >();
// key: MPI processor index 'p'
// values: a set of global column indices, within the range of processor 'p',
// that contain nonzero entries on this processor
int[] col = null;
int L;
// loop over all rows...
for (int i = 0; i < I; i++)
{
L = M.GetOccupiedColumnIndices(i + i0Row, ref col);
// loop over all nonzero entries in the row...
for (int l = 0; l < L; l++)
{
int ColIndex = col[l];
int localColInd = ColIndex - i0Col;
if (localColInd >= 0 && localColInd < J)
{
// column of 'entry' is within the local range of this processor
// + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ColToRowLocal[localColInd].Add(i + i0Row);
}
else
{
// column of 'entry' belongs to external processor 'proc'
// + + + + + + + + + + + + + + + + + + + + + + + + + + + +
int proc = this.ColPart.FindProcess(ColIndex);
{
//SortedDictionary<int, List<int>> ColToRowExt_proc;
//if (!ColToRowExternal.ContainsKey(proc)) {
// ColToRowExt_proc = new SortedDictionary<int, List<int>>();
// ColToRowExternal.Add(proc, ColToRowExt_proc);
//} else {
// ColToRowExt_proc = ColToRowExternal[proc];
//}
int j = ColIndex;
List <int> Rows4Col;
if (!ColToRowExternal.ContainsKey(j))
{
Rows4Col = new List <int>();
ColToRowExternal.Add(j, Rows4Col);
}
else
{
Rows4Col = ColToRowExternal[j];
}
Rows4Col.Add(i + i0Row);
}
{
List <int> ColForProc_proc;
if (!ColForProc.ContainsKey(proc))
{
ColForProc_proc = new List <int>();
ColForProc.Add(proc, ColForProc_proc);
}
else
{
ColForProc_proc = ColForProc[proc];
}
if (!ColForProc_proc.Contains(ColIndex))
{
ColForProc_proc.Add(ColIndex);
}
}
}
}
}
// communicate
// ===========
//SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw.MPI_COMM_WORLD);
//{
// foreach (int proc in ColToRowExternal.Keys) {
// sms.SetCommPath(proc);
// }
// sms.CommitCommPaths();
// // send
// foreach (int proc in ColToRowExternal.Keys) {
// SortedDictionary<int, List<int>> ColToRowExt_proc = ColToRowExternal[proc];
// SortedList _ColToRowExt_proc = new SortedList();
// foreach (int iCol in ColToRowExt_proc.Keys)
// _ColToRowExt_proc.Add(iCol, ColToRowExt_proc[iCol].ToArray());
// }
// // receive
// int p; SortedList rcv;
// sms.GetNext(out p, out rcv);
// while (rcv != null) {
// foreach (int col in rcv.Keys) {
// int[] rowList = (int[])rcv[col];
// ColToRowLocal[col].AddRange(rowList);
// }
// sms.GetNext(out p, out rcv);
// }
//}
//sms.Dispose();
// build 'ColProcessors'
// =====================
//ColProcessors = new List<int>[ColToRowLocal.Length];
//for (int j = 0; j < ColToRowLocal.Length; j++) {
// List<int> mpiRank = null;
// foreach (int rowind in ColToRowLocal[j]) {
// int riloc = rowind - i0Row;
// if (riloc < 0 || riloc >= I) {
// if (mpiRank == null)
// mpiRank = new List<int>();
// mpiRank.Add(M.RowPartiton.FindProcess(rowind));
// }
// ColProcessors[j] = mpiRank;
// }
//}
// communicate: build 'ColProcessors'
// ==================================
{
ColProcessors = new List <int> [ColPart.LocalLength];
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
sms.SetCommPathsAndCommit(ColForProc.Keys);
foreach (int proc in ColForProc.Keys)
{
sms.Transmit(proc, ColForProc[proc].ToArray());
}
int i0Loc = (int)ColPart.i0;
int rcvproc; int[] ColIndices;
while (sms.GetNext(out rcvproc, out ColIndices))
{
int Rank;
{
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out Rank);
//Console.WriteLine("P# " + Rank + ": receiving from P# " + rcvproc);
}
foreach (int ColInd in ColIndices)
{
int localColInd = ColInd - i0Loc;
if (localColInd < 0 || localColInd >= ColPart.LocalLength)
{
throw new IndexOutOfRangeException("internal error");
}
if (ColProcessors[localColInd] == null)
{
ColProcessors[localColInd] = new List <int>();
}
if (ColProcessors[localColInd].Contains(rcvproc))
{
throw new ApplicationException("internal error.");
}
ColProcessors[localColInd].Add(rcvproc);
}
}
sms.Dispose();
}
if (ExtCol != null)
{
var send = new Dictionary <int, List <Tuple <int, List <int> > > >();
int myRank = M.RowPartitioning.MpiRank;
foreach (var kv in ExtCol)
{
int rank = kv.Key;
int[] ColIdx = kv.Value;
var sendToRank = new List <Tuple <int, List <int> > >();
foreach (int iCol in ColIdx)
{
List <int> c2p = ColProcessors[iCol - i0Col];
var t = new Tuple <int, List <int> >(iCol, c2p != null ? new List <int>(c2p) : new List <int>());
t.Item2.Add(myRank);
sendToRank.Add(t);
}
send.Add(rank, sendToRank);
}
var receive = SerialisationMessenger.ExchangeData(send, csMPI.Raw._COMM.WORLD);
ColProcessorsExternal = new Dictionary <int, List <int> >();
foreach (var kv in receive)
{
var val = kv.Value;
foreach (var t in val)
{
int iCol = t.Item1;
List <int> ranks = t.Item2;
int iMyRank = ranks.IndexOf(myRank);
if (iMyRank >= 0)
{
ranks.RemoveAt(iMyRank);
}
ColProcessorsExternal.Add(t.Item1, t.Item2);
Debug.Assert(this.ColPart.FindProcess(t.Item1) == kv.Key);
}
}
#if DEBUG
foreach (var procList in ColProcessorsExternal.Values)
{
Debug.Assert(procList.Contains(myRank) == false);
}
#endif
}
}
/// <summary>
/// Resorts a vector according to this permutation, i.e. the
/// <em>j</em>-th item of the input vector is copied to the
/// <see cref="Values"/>[j]-th entry of the output vector.
/// </summary>
/// <param name="input">
/// Input vector, length must be equal to the length of this permutation, unchanged on exit.
/// </param>
/// <param name="output">
/// On exit, <paramref name="output"/>[<see cref="Values"/>[j]] = <paramref name="input"/>[j]
/// </param>
public void ApplyToVector <I>(IList <I> input, IList <I> output, IPartitioning outputPartitioning)
{
using (new FuncTrace()) {
if (input.Count != this.LocalLength)
{
throw new ArgumentException("wrong size of input vector.");
}
if (output.Count != outputPartitioning.LocalLength)
{
throw new ArgumentException("wrong size of output vector.");
}
long[] TargetInd = this.Values;
// keys: processors which should receive data from this processor
Dictionary <int, ApplyToVector_Helper <I> > sendData =
new Dictionary <int, ApplyToVector_Helper <I> >();
int out_myI0 = outputPartitioning.i0;
int out_nextI0 = out_myI0 + outputPartitioning.LocalLength;
int J = this.Partitioning.LocalLength;
for (int j = 0; j < J; j++)
{
if (out_myI0 <= TargetInd[j] && TargetInd[j] < out_nextI0)
{
// target index located on this processor
output[(int)(TargetInd[j] - out_myI0)] = input[j];
}
else
{
// target index located on other processor
int TargProc = outputPartitioning.FindProcess(TargetInd[j]);
ApplyToVector_Helper <I> sendData_TargProc = null;
if (!sendData.TryGetValue(TargProc, out sendData_TargProc))
{
sendData_TargProc = new ApplyToVector_Helper <I>();
sendData.Add(TargProc, sendData_TargProc);
}
sendData_TargProc.Items.Add(input[j]);
sendData_TargProc.TargetIndices.Add(TargetInd[j]);
}
}
var rcvData = SerialisationMessenger.ExchangeData(
sendData, MPI.Wrappers.csMPI.Raw._COMM.WORLD);
foreach (var rcvPkt in rcvData.Values)
{
int K = rcvPkt.Items.Count;
Debug.Assert(rcvPkt.Items.Count == rcvPkt.TargetIndices.Count);
for (int k = 0; k < K; k++)
{
int locIdx = (int)(rcvPkt.TargetIndices[k]) - out_myI0;
output[locIdx] = rcvPkt.Items[k];
}
}
}
}
