internal override IEnumerable <List <int> > GetBlocking(MultigridOperator op)
{
var MgMap = op.Mapping;
//if(!M.RowPartitioning.Equals(MgMap.Partitioning))
// throw new ArgumentException("Row partitioning mismatch.");
//if(!M.ColPartition.Equals(MgMap.Partitioning))
// throw new ArgumentException("Column partitioning mismatch.");
var ag = MgMap.AggGrid;
int JComp = ag.iLogicalCells.NoOfLocalUpdatedCells; // number of local cells
int[] xadj = new int[JComp + 1];
List <int> adjncy = new List <int>();
for (int j = 0; j < JComp; j++)
{
int[] neigh_j = ag.iLogicalCells.CellNeighbours[j];
foreach (int jNeigh in neigh_j)
{
//adjncy.AddRange(neigh_j);
if (jNeigh < JComp)
{
adjncy.Add(jNeigh);
}
else
{
//Console.WriteLine("Skipping external cell");
}
}
xadj[j + 1] = xadj[j] + neigh_j.Length;
}
int MPIrank, MPIsize;
MPI.Wrappers.csMPI.Raw.Comm_Rank(MPI.Wrappers.csMPI.Raw._COMM.WORLD, out MPIrank);
MPI.Wrappers.csMPI.Raw.Comm_Size(MPI.Wrappers.csMPI.Raw._COMM.WORLD, out MPIsize);
//if (MPIrank == 1)
// NoOfParts = 1;
//Debugger.Launch();
int[] part = new int[JComp];
{
if (NoOfPartsPerProcess > 1)
{
int ncon = 1;
int edgecut = 0;
int[] options = null; //new int[] { 0, 0, 0, 0, 0 };
METIS.PartGraphKway(
ref JComp, ref ncon,
xadj,
adjncy.ToArray(),
null,
null,
null,
ref NoOfPartsPerProcess,
null,
null,
options,
ref edgecut,
part);
}
else
{
part.SetAll(0);
}
}
{
var _Blocks = NoOfPartsPerProcess.ForLoop(i => new List <int>((int)Math.Ceiling(1.1 * JComp / NoOfPartsPerProcess)));
for (int j = 0; j < JComp; j++)
{
_Blocks[part[j]].Add(j);
}
return(_Blocks);
}
}
/// <summary>
/// Computes a grid partitioning (which cell should be on which processor)
/// using the serial METIS library -- work is only done on MPi rank 0.
/// </summary>
/// <param name="cellWeightsLocal">
/// If not null, defines the weight associted with each cell on the current process
/// </param>
/// <param name="noOfPartitioningsToChooseFrom">
/// Tells METIS to compute
/// </param>
/// <returns>
/// Index: local cell index, content: MPI Processor rank;<br/>
/// This is the suggestion
/// of ParMETIS for the grid partitioning:
/// For each local cell index, the returned array contains the MPI
/// process rank where the cell should be placed.
/// </returns>
public int[] ComputePartitionMETIS(int[] cellWeightsLocal = null, int noOfPartitioningsToChooseFrom = 1)
{
using (new FuncTrace()) {
int size = this.Size;
int rank = this.MyRank;
if (size == 1)
{
return(new int[NoOfUpdateCells]);
}
if (this.NumberOfCells_l > int.MaxValue)
{
throw new Exception(String.Format(
"Grid contains more than {0} cells and can thus not be partitioned using METIS. Use ParMETIS instead.",
int.MaxValue));
}
int J = (rank == 0) ? this.NumberOfCells : 0;
// Setup communication; all send to rank 0
SerialisationMessenger sms = new SerialisationMessenger(csMPI.Raw._COMM.WORLD);
if (rank != 0)
{
sms.SetCommPath(0);
}
sms.CommitCommPaths();
// Assemble adjacency lists on rank 0
IEnumerable <Neighbour>[] neighboursGlobal = new IEnumerable <Neighbour> [J];
{
IEnumerable <Neighbour>[] neighboursLocal = GetCellNeighbourship(IncludeBcCells: false).Take(NoOfUpdateCells).ToArray();
if (rank == 0)
{
int localOffset = m_CellPartitioning.GetI0Offest(rank);
int localLength = m_CellPartitioning.GetLocalLength(rank);
for (int i = 0; i < localLength; i++)
{
neighboursGlobal[localOffset + i] = neighboursLocal[i];
}
}
else
{
sms.Transmitt(0, neighboursLocal);
}
while (sms.GetNext(out int senderRank, out IEnumerable <Neighbour>[] neighbours))
{
int localOffset = m_CellPartitioning.GetI0Offest(senderRank);
int localLength = m_CellPartitioning.GetLocalLength(senderRank);
if (neighbours.Length != localLength)
{
throw new Exception();
}
for (int i = 0; i < localLength; i++)
{
neighboursGlobal[localOffset + i] = neighbours[i];
}
}
}
// Gather global weights on rank 0
int[] cellWeightsGlobal = null;
if (cellWeightsLocal != null)
{
cellWeightsGlobal = new int[J];
if (rank == 0)
{
int localOffset = m_CellPartitioning.GetI0Offest(rank);
int localLength = m_CellPartitioning.GetLocalLength(rank);
for (int i = 0; i < localLength; i++)
{
cellWeightsGlobal[localOffset + i] = cellWeightsLocal[i];
}
}
else
{
sms.Transmitt(0, cellWeightsLocal);
}
while (sms.GetNext(out int senderRank, out int[] cellWeights))
{
int localOffset = m_CellPartitioning.GetI0Offest(senderRank);
int localLength = m_CellPartitioning.GetLocalLength(senderRank);
if (cellWeights.Length != localLength)
{
throw new Exception();
}
for (int i = 0; i < localLength; i++)
{
cellWeightsGlobal[localOffset + i] = cellWeights[i];
}
}
}
int[] globalResult = new int[J];
if (rank == 0)
{
int[] xadj = new int[J + 1];
List <int> adjncy = new List <int>(J * m_RefElements[0].NoOfFaces);
for (int j = 0; j < J; j++)
{
var cNj = neighboursGlobal[j];
int E = cNj.Count();
for (int e = 0; e < E; e++)
{
var NN = cNj.ElementAt(e);
if (NN.Neighbour_GlobalIndex >= 0 &&
!NN.IsPeriodicNeighbour)
{
adjncy.Add((int)NN.Neighbour_GlobalIndex);
}
}
xadj[j + 1] = adjncy.Count;
}
// Call METIS
int nparts = size;
Debug.Assert((cellWeightsGlobal == null) == (cellWeightsLocal == null));
int ncon = 1; // One weight per vertex/cell
int objval = -1; // Value of the objective function at return time
int[] Options = new int[METIS.METIS_NOPTIONS];
Options[(int)METIS.OptionCodes.METIS_OPTION_NCUTS] = noOfPartitioningsToChooseFrom; // 5 cuts
Options[(int)METIS.OptionCodes.METIS_OPTION_NITER] = 10; // This is the default refinement iterations
Options[(int)METIS.OptionCodes.METIS_OPTION_UFACTOR] = 30; // Maximum imbalance of 3 percent (this is the default kway clustering)
METIS.ReturnCodes status = (METIS.ReturnCodes)METIS.PartGraphKway(
nvtxs: ref J,
ncon: ref ncon,
xadj: xadj,
adjncy: adjncy.ToArray(),
vwgt: cellWeightsGlobal, // if null, METIS assumes all have weight 1
vsize: null, // No information about communication size
adjwgt: null, // No edge weights
nparts: ref nparts,
tpwgts: null, // No weights for partition constraints
ubvec: null, // No imbalance tolerance for constraints
options: Options,
objval: ref objval,
part: globalResult);
if (status != METIS.ReturnCodes.METIS_OK)
{
throw new Exception(String.Format(
"Error partitioning the mesh. METIS reported {0}",
status));
}
int[] CountCheck = new int[size];
int J2 = this.NumberOfCells;
for (int i = 0; i < J2; i++)
{
CountCheck[globalResult[i]]++;
}
for (int rnk = 0; rnk < size; rnk++)
{
if (CountCheck[rnk] <= 0)
{
throw new ApplicationException("METIS produced illegal partitioning - 0 cells on process " + rnk + ".");
}
}
}
int[] localLengths = new int[size];
for (int p = 0; p < localLengths.Length; p++)
{
localLengths[p] = this.CellPartitioning.GetLocalLength(p);
}
int[] localResult = globalResult.MPIScatterv(localLengths);
return(localResult);
}
}
