public void Redistribute(IDatabaseDriver iom, GridPartType method, string PartOptions) { if (Size <= 1) { return; // nothing to do } throw new NotImplementedException(); }
/// <summary> /// Returns a new grid partition based on the performance model. /// </summary> /// <param name="app"></param> /// <param name="performanceClassCount"></param> /// <param name="cellToPerformanceClassMap"></param> /// <param name="TimestepNo"></param> /// <param name="gridPartType">Grid partitioning type.</param> /// <param name="PartOptions"></param> /// <param name="imbalanceThreshold"> /// See <see cref="Control.AppControl.DynamicLoadBalancing_ImbalanceThreshold"/>. /// </param> /// <param name="Period"> /// See <see cref="Control.AppControl.DynamicLoadBalancing_Period"/>. /// </param> /// <returns></returns> public int[] GetNewPartitioning(IApplication app, int performanceClassCount, int[] cellToPerformanceClassMap, int TimestepNo, GridPartType gridPartType, string PartOptions, double imbalanceThreshold, int Period, bool redistributeAtStartup, TimestepNumber TimestepNoRestart) { // Create new model if number of cell classes has changed for (int i = 0; i < cellCostEstimatorFactories.Count; i++) { if (CurrentCellCostEstimators[i] == null || CurrentCellCostEstimators[i].CurrentPerformanceClassCount != performanceClassCount) { CurrentCellCostEstimators[i] = cellCostEstimatorFactories[i](app, performanceClassCount); } CurrentCellCostEstimators[i].UpdateEstimates(performanceClassCount, cellToPerformanceClassMap); } if (app.Grid.Size == 1) { // only one processor => no load balancing necessary return(null); } bool performPertationing; if (TimestepNo == 0 || (TimestepNoRestart != null && TimestepNo == TimestepNoRestart.MajorNumber)) { performPertationing = redistributeAtStartup; } else { performPertationing = (Period > 0 && TimestepNo % Period == 0); } if (!performPertationing) { return(null); } // No new partitioning if imbalance below threshold double[] imbalanceEstimates = CurrentCellCostEstimators.Select(estimator => estimator.ImbalanceEstimate()).ToArray(); bool imbalanceTooLarge = false; for (int i = 0; i < cellCostEstimatorFactories.Count; i++) { imbalanceTooLarge |= (imbalanceEstimates[i] > imbalanceThreshold); } if (!imbalanceTooLarge) { return(null); } #if DEBUG Console.WriteLine( "At least one runtime imbalance estimate ({0}) was above configured threshold ({1:P1}); attempting repartitioning", String.Join(", ", imbalanceEstimates.Select(e => String.Format("{0:P1}", e))), imbalanceThreshold); #endif IList <int[]> cellCosts = CurrentCellCostEstimators.Select(estimator => estimator.GetEstimatedCellCosts()).ToList(); if (cellCosts == null || cellCosts.All(c => c == null)) { return(null); } if (gridPartType != GridPartType.ParMETIS && gridPartType != GridPartType.Hilbert && cellCosts.Count > 1) { throw new NotImplementedException("Multiple balance constraints only supported using ParMETIS or Hilbert for now"); } int[] result; switch (gridPartType) { case GridPartType.METIS: int.TryParse(PartOptions, out int noOfPartitioningsToChooseFrom); noOfPartitioningsToChooseFrom = Math.Max(1, noOfPartitioningsToChooseFrom); result = ((GridCommons)(app.Grid)).ComputePartitionMETIS(cellCosts.Single()); isFirstRepartitioning = false; break; case GridPartType.ParMETIS: // Do full ParMETIS run on first repartitioning since // initial partitioning may be _really_ bad if (isFirstRepartitioning) { result = ((GridCommons)(app.Grid)).ComputePartitionParMETIS(cellCosts); isFirstRepartitioning = false; } else { // Refinement currently deactivate because it behaves // strangely when large numbers of cells should be // repartitioned //result = Grid.ComputePartitionParMETIS(cellCosts, refineCurrentPartitioning: true); result = ((GridCommons)(app.Grid)).ComputePartitionParMETIS(cellCosts); } break; case GridPartType.Hilbert: return(((GridCommons)(app.Grid)).ComputePartitionHilbert(localcellCosts: cellCosts, Functype: 0)); case GridPartType.directHilbert: return(((GridCommons)(app.Grid)).ComputePartitionHilbert(localcellCosts: cellCosts, Functype: 1)); case GridPartType.none: result = IndexBasedPartition(cellCosts.Single()); break; case GridPartType.Predefined: return(null); default: throw new NotImplementedException(); } if (result.Length == 0) { throw new Exception(String.Format( "LoadBalancer computed invalid partitioning; no cells left on rank {0}", app.Grid.MyRank)); } return(result); }
/// <summary> /// Distributes cells to processors by using ParMETIS; /// Can't be used after <see cref="GridData"/> object is constructed. /// </summary> public void Redistribute(IDatabaseDriver iom, GridPartType method, string PartOptions) { using (new FuncTrace()) { ilPSP.MPICollectiveWatchDog.Watch(MPI.Wrappers.csMPI.Raw._COMM.WORLD); int size; int rank; csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out rank); csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out size); //// invalid from now on //m_GlobalId2CellIndexMap = null; int[] part; switch (method) { case GridPartType.METIS: if (size > 1) { int.TryParse(PartOptions, out int noOfPartitioningsToChooseFrom); noOfPartitioningsToChooseFrom = Math.Max(1, noOfPartitioningsToChooseFrom); part = ComputePartitionMETIS(noOfPartitioningsToChooseFrom: noOfPartitioningsToChooseFrom); RedistributeGrid(part); } break; case GridPartType.ParMETIS: if (size > 1) { int.TryParse(PartOptions, out int noOfRefinements); part = ComputePartitionParMETIS(); RedistributeGrid(part); for (int i = 0; i < noOfRefinements; i++) { part = ComputePartitionParMETIS(refineCurrentPartitioning: true); RedistributeGrid(part); } } break; case GridPartType.Hilbert: part = ComputePartitionHilbert(); RedistributeGrid(part); break; case GridPartType.none: break; case GridPartType.Predefined: if (size > 1) { if (PartOptions == null || PartOptions.Length <= 0) { //throw new ArgumentException("'" + GridPartType.Predefined.ToString() + "' requires, as an option, the name of the Partition.", "PartOptions"); PartOptions = size.ToString(); } if (!m_PredefinedGridPartitioning.ContainsKey(PartOptions)) { StringWriter stw = new StringWriter(); for (int i = 0; i < m_PredefinedGridPartitioning.Count; i++) { stw.Write("'" + m_PredefinedGridPartitioning.Keys[i] + "'"); if (i < (m_PredefinedGridPartitioning.Count - 1)) { stw.Write(", "); } } throw new ArgumentException("Grid Partitioning with name '" + PartOptions + "' is unknown; known are: " + stw.ToString() + ";"); } Console.WriteLine("redistribution according to " + PartOptions); var partHelp = m_PredefinedGridPartitioning[PartOptions]; part = partHelp.CellToRankMap; if (part == null) { var cp = this.CellPartitioning; part = iom.LoadVector <int>(partHelp.Guid, ref cp).ToArray(); } int Min = int.MinValue; int Max = int.MaxValue; { int LocMin = 0; int LocMax = 0; for (int j = part.Length - 1; j >= 0; j--) { LocMin = Math.Min(LocMin, part[j]); LocMax = Math.Max(LocMax, part[j]); } unsafe { csMPI.Raw.Allreduce((IntPtr)(&LocMin), (IntPtr)(&Min), 1, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.MIN, csMPI.Raw._COMM.WORLD); csMPI.Raw.Allreduce((IntPtr)(&LocMax), (IntPtr)(&Max), 1, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.MAX, csMPI.Raw._COMM.WORLD); } } if (Min < 0) { throw new ApplicationException("illegal predefined partition: minimum processor ranks is " + Min + ";"); } if (Max >= size) { throw new ApplicationException("predefined partition not usable: specifies " + (Max + 1) + " processors, but currently running on " + size + " processors."); } RedistributeGrid(part); } break; default: throw new NotImplementedException(); } csMPI.Raw.Barrier(csMPI.Raw._COMM.WORLD); } }