/// <summary> /// returns all external rows of <paramref name="M"/> /// corresponding to ghost cells of <paramref name="map"/>, /// which are located on other Mpi-ranks. /// </summary> /// <param name="map">Multigrid mapping</param> /// <param name="M">matrix distributed according to <paramref name="map"/></param> /// <returns></returns> public static BlockMsrMatrix GetAllExternalRows(MultigridMapping map, BlockMsrMatrix M) { var extcells = map.AggGrid.iLogicalCells.NoOfExternalCells.ForLoop(i => i + map.LocalNoOfBlocks); var SBS = new SubBlockSelector(map); SBS.CellSelector(extcells, false); var AllExtMask = new BlockMaskExt(SBS, 0); var ExternalRows_BlockI0 = AllExtMask.GetAllSubMatrixCellOffsets(); var ExternalRows_BlockN = AllExtMask.GetAllSubMatrixCellLength(); var ExternalRowsIndices = AllExtMask.m_GlobalMask; BlockPartitioning PermRow = new BlockPartitioning(ExternalRowsIndices.Count, ExternalRows_BlockI0, ExternalRows_BlockN, M.MPI_Comm, i0isLocal: true); BlockMsrMatrix Perm = new BlockMsrMatrix(PermRow, M._RowPartitioning); for (int iRow = 0; iRow < ExternalRowsIndices.Count; iRow++) { Debug.Assert(M._RowPartitioning.IsInLocalRange(ExternalRowsIndices[iRow]) == false); Perm[iRow + PermRow.i0, ExternalRowsIndices[iRow]] = 1; } #if TEST Perm.SaveToTextFileSparseDebug("Perm"); #endif return(BlockMsrMatrix.Multiply(Perm, M)); }
public BlockMaskLoc(SubBlockSelector sbs) : base(sbs, csMPI.Raw._COMM.SELF) { base.GenerateAllMasks(); foreach (var idx in this.m_GlobalMask) { Debug.Assert(idx >= m_map.i0); Debug.Assert(idx < m_map.iE); } }
private void AssignXdgBlocksModification(SubBlockSelector sbs, MultigridOperator op, bool IsLowSelector) { var Filter = sbs.ModeFilter; Func <int, int, int, int, bool> Modification = delegate(int iCell, int iVar, int iSpec, int pDeg) { int NoOfSpec = op.Mapping.AggBasis[0].GetNoOfSpecies(iCell); if (NoOfSpec >= 2) { return(IsLowSelector); } else { return(Filter(iCell, iVar, iSpec, pDeg)); } }; sbs.ModeSelector(Modification); }
/// <summary> /// Generates Block Mask (index lists) from Sub block selection based on a multigrid mapping. /// abstract parts are individuallized by child classes: <see cref="BlockMask.BlockMaskLoc"/> and <see cref="BlockMask.BlockMaskExt"/> /// </summary> /// <param name="SBS"></param> public BlockMaskBase(SubBlockSelector SBS, MPI_Comm MPIcomm) { m_map = SBS.GetMapping; m_sbs = SBS; m_AggBS = m_map.AggBasis; m_DGdegree = m_map.DgDegree; m_Ni0 = Ni0Gen(); m_NoOfVariables = m_AggBS.Length; //Testen ob es cells gibt wo Var<>NoOfVar, das würde dementsprechend auch m_DG beeinflussen m_NoOfSpecies = new int[m_NoOfCells][]; for (int iCell = 0; iCell < m_NoOfCells; iCell++) { m_NoOfSpecies[iCell] = new int[m_NoOfVariables]; for (int iVar = 0; iVar < m_NoOfVariables; iVar++) { m_NoOfSpecies[iCell][iVar] = m_AggBS[iVar].GetNoOfSpecies(iCell + m_CellOffset); } } }
/// <summary> /// generates a masking of subblocks within a <see cref="MultigridMapping"/> according to <see cref="SubBlockSelector"/> <paramref name="sbs"/> /// enables applying mask onto matrices and vectors, which comply with this <see cref="MultigridMapping"/>. /// (e.g. sub matrix generation, sub vector extraction, etc.). /// The smallest unit are dg blocks. /// The masking operates on local blocks available on this proc per default, /// which can be exceeded to external blocks by providing external rows: <paramref name="ExtRows"/>. /// Ghost cells (covert by <see cref="BoSSS.Foundation.Grid.ILogicalCellData.NoOfExternalCells"/>) can be acquired by <see cref="GetAllExternalRows"/>. /// </summary> /// <param name="sbs">sub block selection defined by dev</param> /// <param name="ExtRows">external rows collected from other MPI-processes on this proc</param> public BlockMask(SubBlockSelector sbs, BlockMsrMatrix ExtRows = null) { m_map = sbs.GetMapping; m_ExtRows = ExtRows; m_includeExternalCells = (ExtRows != null) && m_map.MpiSize > 1; BMLoc = new BlockMaskLoc(sbs); if (m_includeExternalCells) { BMExt = new BlockMaskExt(sbs, BMLoc.LocalDOF); SetThisShitUp(new BlockMaskBase[] { BMLoc, BMExt }); } else { SetThisShitUp(new BlockMaskBase[] { BMLoc }); } #if Debug CheckIndices(); #endif }
public BlockMaskExt(SubBlockSelector SBS, int LocMaskOffset) : base(SBS, SBS.GetMapping.MPI_Comm) { // must be set before mask generation m_LocOffset = LocMaskOffset; m_extLocLen = m_map.GetLocalLength_Ext(); base.GenerateAllMasks(); foreach (int idx in this.m_GlobalMask) { Debug.Assert(idx < m_map.i0 || idx >= m_map.iE); } int LL = m_map.LocalLength; int jMax = m_map.AggGrid.iLogicalCells.Count - 1; int LE = m_map.LocalUniqueIndex(0, jMax, 0) + m_map.GetLength(jMax); foreach (int idx in this.m_LocalMask) { Debug.Assert(idx >= LL); Debug.Assert(idx < LE); } }
private void ModifyHighSelector(SubBlockSelector sbs, MultigridOperator op) { AssignXdgBlocksModification(sbs, op, false); }
private void ModifyLowSelector(SubBlockSelector sbs, MultigridOperator op) { AssignXdgBlocksModification(sbs, op, true); }
/// <summary> /// Krankplätze müssen verdichtet werden /// -Kranführer Ronny, ProSieben Reportage /// </summary> public void Init(MultigridOperator op) { // //System.Threading.Thread.Sleep(10000); // //ilPSP.Environment.StdoutOnlyOnRank0 = false; m_op = op; if (m_CoarseLowOrder > m_op.Mapping.DgDegree.Max()) { throw new ArgumentOutOfRangeException("CoarseLowOrder is higher than maximal DG degree"); } #if TEST var debugerSW = new StreamWriter(String.Concat("debug_of_", ilPSP.Environment.MPIEnv.MPI_Rank)); Console.WriteLine("variable TEST is defined"); //debugerSW.WriteLine("proc {0} reporting Num of Blocks {1}", ilPSP.Environment.MPIEnv.MPI_Rank, HighOrderBlocks_LUpivots.Length); #endif int D = this.m_op.GridData.SpatialDimension; var DGlowSelect = new SubBlockSelector(op.Mapping); Func <int, int, int, int, bool> lowFilter = (int iCell, int iVar, int iSpec, int pDeg) => pDeg <= (iVar != D ? CoarseLowOrder : CoarseLowOrder - 1); DGlowSelect.ModeSelector(lowFilter); if (AssignXdGCellsToLowBlocks) { ModifyLowSelector(DGlowSelect, op); } lMask = new BlockMask(DGlowSelect); m_lowMaskLen = lMask.GetNoOfMaskedRows; if (UseHiOrderSmoothing) { var DGhighSelect = new SubBlockSelector(op.Mapping); Func <int, int, int, int, bool> highFilter = (int iCell, int iVar, int iSpec, int pDeg) => pDeg > (iVar != D ? CoarseLowOrder : CoarseLowOrder - 1); DGhighSelect.ModeSelector(highFilter); if (AssignXdGCellsToLowBlocks) { ModifyHighSelector(DGhighSelect, op); } hMask = new BlockMask(DGhighSelect); m_highMaskLen = hMask.GetNoOfMaskedRows; BlockMsrMatrix P01HiMatrix = null; if (UseDiagonalPmg) { HighOrderBlocks_LU = hMask.GetDiagonalBlocks(op.OperatorMatrix, false, false); int NoOfBlocks = HighOrderBlocks_LU.Length; HighOrderBlocks_LUpivots = new int[NoOfBlocks][]; for (int jLoc = 0; jLoc < NoOfBlocks; jLoc++) { int len = HighOrderBlocks_LU[jLoc].NoOfRows; HighOrderBlocks_LUpivots[jLoc] = new int[len]; HighOrderBlocks_LU[jLoc].FactorizeLU(HighOrderBlocks_LUpivots[jLoc]); } } else { P01HiMatrix = hMask.GetSubBlockMatrix(op.OperatorMatrix, csMPI.Raw._COMM.SELF); hiSolver = new PARDISOSolver() { CacheFactorization = true, UseDoublePrecision = true, // keep it true, experiments showed, that this leads to fewer iterations SolverVersion = Parallelism.OMP }; hiSolver.DefineMatrix(P01HiMatrix); } } var P01SubMatrix = lMask.GetSubBlockMatrix(op.OperatorMatrix, csMPI.Raw._COMM.WORLD); intSolver = new PARDISOSolver() { CacheFactorization = true, UseDoublePrecision = false, // no difference towards =true observed for XDGPoisson SolverVersion = Parallelism.OMP }; intSolver.DefineMatrix(P01SubMatrix); #if TEST P01SubMatrix.SaveToTextFileSparseDebug("lowM"); P01SubMatrix.SaveToTextFileSparse("lowM_full"); if (!UseDiagonalPmg) { //P01HiMatrix.SaveToTextFileSparseDebug("hiM"); //P01HiMatrix.SaveToTextFileSparse("hiM_full"); } m_op.OperatorMatrix.SaveToTextFileSparseDebug("M"); m_op.OperatorMatrix.SaveToTextFileSparse("M_full"); debugerSW.Flush(); debugerSW.Close(); long[] bla = m_op.BaseGridProblemMapping.GridDat.CurrentGlobalIdPermutation.Values; bla.SaveToTextFileDebug("permutation_"); List <int> BlockI0 = new List <int>(); List <int> Block_N = new List <int>(); foreach (long Block in bla) { BlockI0.Add(m_op.Mapping.GetBlockI0((int)Block)); Block_N.Add(m_op.Mapping.GetBlockLen((int)Block)); } BlockI0.SaveToTextFileDebug("BlockI0"); Block_N.SaveToTextFileDebug("Block_N"); #endif }