public void Redistribute(IDatabaseDriver iom, GridPartType method, string PartOptions)
{
if (Size <= 1)
{
return; // nothing to do
}
throw new NotImplementedException();
}
/// <summary>
/// Returns a new grid partition based on the performance model.
/// </summary>
/// <param name="app"></param>
/// <param name="performanceClassCount"></param>
/// <param name="cellToPerformanceClassMap"></param>
/// <param name="TimestepNo"></param>
/// <param name="gridPartType">Grid partitioning type.</param>
/// <param name="PartOptions"></param>
/// <param name="imbalanceThreshold">
/// See <see cref="Control.AppControl.DynamicLoadBalancing_ImbalanceThreshold"/>.
/// </param>
/// <param name="Period">
/// See <see cref="Control.AppControl.DynamicLoadBalancing_Period"/>.
/// </param>
/// <returns></returns>
public int[] GetNewPartitioning(IApplication app, int performanceClassCount, int[] cellToPerformanceClassMap, int TimestepNo, GridPartType gridPartType, string PartOptions, double imbalanceThreshold, int Period, bool redistributeAtStartup, TimestepNumber TimestepNoRestart)
{
// Create new model if number of cell classes has changed
for (int i = 0; i < cellCostEstimatorFactories.Count; i++)
{
if (CurrentCellCostEstimators[i] == null ||
CurrentCellCostEstimators[i].CurrentPerformanceClassCount != performanceClassCount)
{
CurrentCellCostEstimators[i] = cellCostEstimatorFactories[i](app, performanceClassCount);
}
CurrentCellCostEstimators[i].UpdateEstimates(performanceClassCount, cellToPerformanceClassMap);
}
if (app.Grid.Size == 1)
{
// only one processor => no load balancing necessary
return(null);
}
bool performPertationing;
if (TimestepNo == 0 || (TimestepNoRestart != null && TimestepNo == TimestepNoRestart.MajorNumber))
{
performPertationing = redistributeAtStartup;
}
else
{
performPertationing = (Period > 0 && TimestepNo % Period == 0);
}
if (!performPertationing)
{
return(null);
}
// No new partitioning if imbalance below threshold
double[] imbalanceEstimates =
CurrentCellCostEstimators.Select(estimator => estimator.ImbalanceEstimate()).ToArray();
bool imbalanceTooLarge = false;
for (int i = 0; i < cellCostEstimatorFactories.Count; i++)
{
imbalanceTooLarge |= (imbalanceEstimates[i] > imbalanceThreshold);
}
if (!imbalanceTooLarge)
{
return(null);
}
#if DEBUG
Console.WriteLine(
"At least one runtime imbalance estimate ({0}) was above configured threshold ({1:P1}); attempting repartitioning",
String.Join(", ", imbalanceEstimates.Select(e => String.Format("{0:P1}", e))),
imbalanceThreshold);
#endif
IList <int[]> cellCosts = CurrentCellCostEstimators.Select(estimator => estimator.GetEstimatedCellCosts()).ToList();
if (cellCosts == null || cellCosts.All(c => c == null))
{
return(null);
}
if (gridPartType != GridPartType.ParMETIS && gridPartType != GridPartType.Hilbert && cellCosts.Count > 1)
{
throw new NotImplementedException("Multiple balance constraints only supported using ParMETIS or Hilbert for now");
}
int[] result;
switch (gridPartType)
{
case GridPartType.METIS:
int.TryParse(PartOptions, out int noOfPartitioningsToChooseFrom);
noOfPartitioningsToChooseFrom = Math.Max(1, noOfPartitioningsToChooseFrom);
result = ((GridCommons)(app.Grid)).ComputePartitionMETIS(cellCosts.Single());
isFirstRepartitioning = false;
break;
case GridPartType.ParMETIS:
// Do full ParMETIS run on first repartitioning since
// initial partitioning may be _really_ bad
if (isFirstRepartitioning)
{
result = ((GridCommons)(app.Grid)).ComputePartitionParMETIS(cellCosts);
isFirstRepartitioning = false;
}
else
{
// Refinement currently deactivate because it behaves
// strangely when large numbers of cells should be
// repartitioned
//result = Grid.ComputePartitionParMETIS(cellCosts, refineCurrentPartitioning: true);
result = ((GridCommons)(app.Grid)).ComputePartitionParMETIS(cellCosts);
}
break;
case GridPartType.Hilbert:
return(((GridCommons)(app.Grid)).ComputePartitionHilbert(localcellCosts: cellCosts, Functype: 0));
case GridPartType.directHilbert:
return(((GridCommons)(app.Grid)).ComputePartitionHilbert(localcellCosts: cellCosts, Functype: 1));
case GridPartType.none:
result = IndexBasedPartition(cellCosts.Single());
break;
case GridPartType.Predefined:
return(null);
default:
throw new NotImplementedException();
}
if (result.Length == 0)
{
throw new Exception(String.Format(
"LoadBalancer computed invalid partitioning; no cells left on rank {0}",
app.Grid.MyRank));
}
return(result);
}
/// <summary>
/// Distributes cells to processors by using ParMETIS;
/// Can't be used after <see cref="GridData"/> object is constructed.
/// </summary>
public void Redistribute(IDatabaseDriver iom, GridPartType method, string PartOptions)
{
using (new FuncTrace()) {
ilPSP.MPICollectiveWatchDog.Watch(MPI.Wrappers.csMPI.Raw._COMM.WORLD);
int size;
int rank;
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out rank);
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out size);
//// invalid from now on
//m_GlobalId2CellIndexMap = null;
int[] part;
switch (method)
{
case GridPartType.METIS:
if (size > 1)
{
int.TryParse(PartOptions, out int noOfPartitioningsToChooseFrom);
noOfPartitioningsToChooseFrom = Math.Max(1, noOfPartitioningsToChooseFrom);
part = ComputePartitionMETIS(noOfPartitioningsToChooseFrom: noOfPartitioningsToChooseFrom);
RedistributeGrid(part);
}
break;
case GridPartType.ParMETIS:
if (size > 1)
{
int.TryParse(PartOptions, out int noOfRefinements);
part = ComputePartitionParMETIS();
RedistributeGrid(part);
for (int i = 0; i < noOfRefinements; i++)
{
part = ComputePartitionParMETIS(refineCurrentPartitioning: true);
RedistributeGrid(part);
}
}
break;
case GridPartType.Hilbert:
part = ComputePartitionHilbert();
RedistributeGrid(part);
break;
case GridPartType.none:
break;
case GridPartType.Predefined:
if (size > 1)
{
if (PartOptions == null || PartOptions.Length <= 0)
{
//throw new ArgumentException("'" + GridPartType.Predefined.ToString() + "' requires, as an option, the name of the Partition.", "PartOptions");
PartOptions = size.ToString();
}
if (!m_PredefinedGridPartitioning.ContainsKey(PartOptions))
{
StringWriter stw = new StringWriter();
for (int i = 0; i < m_PredefinedGridPartitioning.Count; i++)
{
stw.Write("'" + m_PredefinedGridPartitioning.Keys[i] + "'");
if (i < (m_PredefinedGridPartitioning.Count - 1))
{
stw.Write(", ");
}
}
throw new ArgumentException("Grid Partitioning with name '" + PartOptions + "' is unknown; known are: " + stw.ToString() + ";");
}
Console.WriteLine("redistribution according to " + PartOptions);
var partHelp = m_PredefinedGridPartitioning[PartOptions];
part = partHelp.CellToRankMap;
if (part == null)
{
var cp = this.CellPartitioning;
part = iom.LoadVector <int>(partHelp.Guid, ref cp).ToArray();
}
int Min = int.MinValue;
int Max = int.MaxValue;
{
int LocMin = 0;
int LocMax = 0;
for (int j = part.Length - 1; j >= 0; j--)
{
LocMin = Math.Min(LocMin, part[j]);
LocMax = Math.Max(LocMax, part[j]);
}
unsafe {
csMPI.Raw.Allreduce((IntPtr)(&LocMin), (IntPtr)(&Min), 1, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.MIN, csMPI.Raw._COMM.WORLD);
csMPI.Raw.Allreduce((IntPtr)(&LocMax), (IntPtr)(&Max), 1, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.MAX, csMPI.Raw._COMM.WORLD);
}
}
if (Min < 0)
{
throw new ApplicationException("illegal predefined partition: minimum processor ranks is " + Min + ";");
}
if (Max >= size)
{
throw new ApplicationException("predefined partition not usable: specifies " + (Max + 1) + " processors, but currently running on " + size + " processors.");
}
RedistributeGrid(part);
}
break;
default:
throw new NotImplementedException();
}
csMPI.Raw.Barrier(csMPI.Raw._COMM.WORLD);
}
}
