Exemplo n.º 1
0
        /// <summary>
        /// returns all external rows of <paramref name="M"/>
        /// corresponding to ghost cells of <paramref name="map"/>,
        /// which are located on other Mpi-ranks.
        /// </summary>
        /// <param name="map">Multigrid mapping</param>
        /// <param name="M">matrix distributed according to <paramref name="map"/></param>
        /// <returns></returns>
        public static BlockMsrMatrix GetAllExternalRows(MultigridMapping map, BlockMsrMatrix M)
        {
            var extcells = map.AggGrid.iLogicalCells.NoOfExternalCells.ForLoop(i => i + map.LocalNoOfBlocks);

            var SBS = new SubBlockSelector(map);

            SBS.CellSelector(extcells, false);
            var AllExtMask = new BlockMaskExt(SBS, 0);

            var ExternalRows_BlockI0 = AllExtMask.GetAllSubMatrixCellOffsets();
            var ExternalRows_BlockN  = AllExtMask.GetAllSubMatrixCellLength();
            var ExternalRowsIndices  = AllExtMask.m_GlobalMask;

            BlockPartitioning PermRow = new BlockPartitioning(ExternalRowsIndices.Count, ExternalRows_BlockI0, ExternalRows_BlockN, M.MPI_Comm, i0isLocal: true);

            BlockMsrMatrix Perm = new BlockMsrMatrix(PermRow, M._RowPartitioning);

            for (int iRow = 0; iRow < ExternalRowsIndices.Count; iRow++)
            {
                Debug.Assert(M._RowPartitioning.IsInLocalRange(ExternalRowsIndices[iRow]) == false);
                Perm[iRow + PermRow.i0, ExternalRowsIndices[iRow]] = 1;
            }

#if TEST
            Perm.SaveToTextFileSparseDebug("Perm");
#endif
            return(BlockMsrMatrix.Multiply(Perm, M));
        }
Exemplo n.º 2
0
        /// <summary>
        /// Get SubMatrix corresponding to this <see cref="BlockMask"/>.
        /// With the ignore flags, coupling blocks can be left out (e.g. blocks containing level-set).
        /// If <paramref name="ignoreCellCoupling"/> is set true, only diagonal blocks are concidered.
        /// Probably slower than <see cref="GetSubBlockMatrix(BlockMsrMatrix)"/>.
        /// </summary>
        /// <param name="source">matrix to apply masking to</param>
        /// <param name="ignoreCellCoupling">flag to ignore cell coupling</param>
        /// <param name="ignoreVarCoupling">flag to ignore variable coupling</param>
        /// <param name="ignoreSpecCoupling">flag to ignore species coupling</param>
        /// <returns>sub block matrix</returns>
        public BlockMsrMatrix GetSubBlockMatrix(BlockMsrMatrix source, bool ignoreCellCoupling, bool ignoreVarCoupling, bool ignoreSpecCoupling)
        {
            if (ignoreCellCoupling && m_includeExternalCells)
            {
                throw new NotImplementedException("Coupling of internal and external block is not concidered");
            }

            BlockMsrMatrix submatrix = null;

            if (m_includeExternalCells)
            {
                BlockPartitioning extBlocking = new BlockPartitioning(BMLoc.LocalDOF + BMExt.LocalDOF, SubMatrixOffsets, SubMatrixLen, csMPI.Raw._COMM.SELF);
                submatrix = new BlockMsrMatrix(extBlocking);
                var ExtRowsTmp = m_ExtRows;
                AuxGetSubBlockMatrix(submatrix, source, BMLoc, ignoreCellCoupling, ignoreVarCoupling, ignoreSpecCoupling);
                AuxGetSubBlockMatrix(submatrix, ExtRowsTmp, BMExt, ignoreCellCoupling, ignoreVarCoupling, ignoreSpecCoupling);
            }
            else
            {
                int Loclength = BMLoc.LocalDOF;
                var tmpN      = BMLoc.GetAllSubMatrixCellLength();
                var tmpi0     = BMLoc.GetAllSubMatrixCellOffsets();
                BlockPartitioning localBlocking = new BlockPartitioning(Loclength, tmpi0.ToArray(), tmpN.ToArray(), csMPI.Raw._COMM.SELF, i0isLocal: true);
                submatrix = new BlockMsrMatrix(localBlocking);
                AuxGetSubBlockMatrix(submatrix, source, BMLoc, ignoreCellCoupling, ignoreVarCoupling, ignoreSpecCoupling);
            }
            Debug.Assert(submatrix != null);
            return(submatrix);
        }
Exemplo n.º 3
0
        /// <summary>
        /// If you just want to get the <see cref="BlockMsrMatrix"/>, which corresponds to this <see cref="BlockMask"/>.
        /// This is the method to choose! In addition, MPI communicator can be defined via <paramref name="comm"/>.
        /// </summary>
        /// <returns>submatrix on <paramref name="comm"/></returns>
        public BlockMsrMatrix GetSubBlockMatrix(BlockMsrMatrix source, MPI_Comm comm)
        {
            if (source == null)
            {
                throw new ArgumentNullException();
            }
            if (source.NoOfRows < BMLoc.LocalDOF)
            {
                throw new ArgumentException();
            }

            BlockMsrMatrix target;

            if (m_includeExternalCells)
            {
                BlockPartitioning targetBlocking = new BlockPartitioning(BMLoc.LocalDOF + BMExt.LocalDOF, SubMatrixOffsets, SubMatrixLen, comm);

                //make an extended block dummy to fit local and external blocks
                target = new BlockMsrMatrix(targetBlocking, targetBlocking);

                //get the external rows via MPI exchange
                var ExtRowsTmp = m_ExtRows;

                int offset       = BMLoc.m_GlobalMask.Count;
                var extBlockRows = BMExt.m_GlobalMask.Count.ForLoop(i => i + offset);
                var extBlockCols = BMExt.m_GlobalMask.Count.ForLoop(i => i + offset);

                //ExtRowsTmp lives at the MPI-Communicator of the target, thus the global index is related to a new partitioning and has nothing to do with the partitioning of the multigrid operator ...

                var GlobalIdxExtRows = BMExt.m_GlobalMask.Count.ForLoop(i => BMExt.m_LocalMask[i] - m_map.LocalLength);
                for (int iGlob = 0; iGlob < GlobalIdxExtRows.Count(); iGlob++)
                {
                    Debug.Assert(GlobalIdxExtRows[iGlob] < ExtRowsTmp._RowPartitioning.LocalLength);
                    GlobalIdxExtRows[iGlob] += ExtRowsTmp._RowPartitioning.i0;
                    Debug.Assert(ExtRowsTmp._RowPartitioning.IsInLocalRange(GlobalIdxExtRows[iGlob]));
                }

                //add local Block ...
                source.WriteSubMatrixTo(target, BMLoc.m_GlobalMask, default(int[]), BMLoc.m_GlobalMask, default(int[]));

                //add columns related to external rows ...
                source.AccSubMatrixTo(1.0, target, BMLoc.m_GlobalMask, default(int[]), new int[0], default(int[]), BMExt.m_GlobalMask, extBlockCols);

                //add external rows ...
                ExtRowsTmp.AccSubMatrixTo(1.0, target, GlobalIdxExtRows, extBlockRows, BMLoc.m_GlobalMask, default(int[]), BMExt.m_GlobalMask, extBlockCols);
            }
            else
            {
                BlockPartitioning localBlocking = new BlockPartitioning(BMLoc.LocalDOF, SubMatrixOffsets, SubMatrixLen, csMPI.Raw._COMM.SELF, i0isLocal: true);
                target = new BlockMsrMatrix(localBlocking);
                source.AccSubMatrixTo(1.0, target, BMLoc.m_GlobalMask, default(int[]), BMLoc.m_GlobalMask, default(int[]));
            }
            Debug.Assert(target != null);
            return(target);
        }
Exemplo n.º 4
0
        public static BlockMsrMatrix GetCellCompMatrix(SelectionType SType, MultigridOperator mop, int iB)
        {
            int rank   = mop.Mapping.MpiRank;
            int iBlock = mop.Mapping.AggGrid.CellPartitioning.i0 + iB;
            int i0     = mop.Mapping.GetBlockI0(iBlock);

            //int jBlock = i0 + jB;
            int R = mop.Mapping.GetBlockLen(iBlock);
            //int C = mop.Mapping.GetBlockLen(jBlock);

            bool ZwoSpecR = Math.Max(mop.Mapping.GetSubblkLen(0)[0], mop.Mapping.GetSubblkLen(1)[0]) == R;
            //bool ZwoSpecC = (mop.Mapping.AggBasis[0].GetMaximalLength(DGdegree) + mop.Mapping.AggBasis[0].GetMaximalLength(DGdegree - 1)) == C;

            SpeciesId A       = ((XdgAggregationBasis)mop.Mapping.AggBasis[0]).UsedSpecies[0];
            int       Specpos = ((XdgAggregationBasis)mop.Mapping.AggBasis[0]).GetSpeciesIndex(iB, A);

            int[] SubIdcR = GetSubIndices(SType, ZwoSpecR, Specpos);
            //int[] SubIdcC = GetSubIndices(SType, ZwoSpecC);

            for (int i = 0; i < SubIdcR.Length; i++)
            {
                Debug.Assert(SubIdcR[i] < R);
                SubIdcR[i] += i0;
            }
            //for (int i = 0; i < SubIdcC.Length; i++) {
            //    Debug.Assert(SubIdcC[i] < C);
            //    SubIdcC[i] += i0;
            //}
            //return mop.OperatorMatrix.GetSubMatrix(SubIdcR, SubIdcC);
            var part = new BlockPartitioning(SubIdcR.Length, new int[] { 0 }, new int[] { SubIdcR.Length }, csMPI.Raw._COMM.SELF);

            BlockMsrMatrix sub = new BlockMsrMatrix(part);

            mop.OperatorMatrix.WriteSubMatrixTo(sub, SubIdcR, default(int[]), SubIdcR, default(int[]));
            return(sub);

            //return mop.OperatorMatrix.GetSubMatrix(SubIdcR, SubIdcR);
        }
Exemplo n.º 5
0
        public static BlockMsrMatrix ConvertToQuadraticBMsr(this BlockMsrMatrix M, int[] Colidx, bool isinternal)
        {
            Debug.Assert(M._RowPartitioning.LocalLength == Colidx.Length);

            int NoOfBlocks = M._RowPartitioning.LocalNoOfBlocks;

            int[] Offsets      = new int[NoOfBlocks];
            int[] Lengths      = new int[NoOfBlocks];
            int   IdxOffset    = M._RowPartitioning.i0;
            int   ColIdxOffset = M._ColPartitioning.i0;

            for (int i = 0; i < NoOfBlocks; i++)
            {
                int iBlock = i + M._RowPartitioning.FirstBlock;
                Offsets[i] = M._RowPartitioning.GetBlockI0(iBlock) - IdxOffset;
                Lengths[i] = M._RowPartitioning.GetBlockLen(iBlock);
            }
            BlockPartitioning part = new BlockPartitioning(M._RowPartitioning.LocalLength, Offsets, Lengths, csMPI.Raw._COMM.SELF, true);
            BlockMsrMatrix    ret  = new BlockMsrMatrix(part);

            int[] RowISrc = M._RowPartitioning.LocalLength.ForLoop(i => i + IdxOffset);
            //int[] ColISrc = M._ColPartitioning.LocalLength.ForLoop(i => Colidx[i]);
            //if (ColISrc.Length < RowISrc.Length)
            //    ExtISrc = (RowISrc.Length - ColISrc.Length).ForLoop(i => Colidx[i+ ColISrc.Length]);
            int[] ExtISrc = M._RowPartitioning.LocalLength.ForLoop(i => Colidx[i]);
            int[] ExtITrg = M._RowPartitioning.LocalLength.ForLoop(i => i);
            if (isinternal)
            {
                M.AccSubMatrixTo(1.0, ret, RowISrc, default(int[]), ExtISrc, default(int[]));
            }
            else
            {
                M.AccSubMatrixTo(1.0, ret, RowISrc, default(int[]), new int[0], default(int[]), ExtISrc, ExtITrg);
            }

            return(ret);
        }
Exemplo n.º 6
0
        public void Init(MultigridOperator op)
        {
            using (new FuncTrace()) {
                if (m_MgOp != null)
                {
                    // ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
                    // someone is trying to re-use this solver: see if the settings permit that
                    // ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

                    if (op.LevelIndex != m_MgOp.LevelIndex)
                    {
                        throw new ArgumentException("Re-use on different level not possible.");
                    }
                    if (!this.MtxFull._RowPartitioning.EqualsPartition(op.OperatorMatrix._RowPartitioning))
                    {
                        throw new ArgumentException("Matrix has changed, unable to re-use");
                    }
                    if (!this.MtxFull._ColPartitioning.EqualsPartition(op.OperatorMatrix._ColPartitioning))
                    {
                        throw new ArgumentException("Matrix has changed, unable to re-use");
                    }
#if DEBUG
                    if (!object.ReferenceEquals(this.MtxFull, op.OperatorMatrix))
                    {
                        BlockMsrMatrix Check = this.MtxFull.CloneAs();
                        Check.Acc(-1.0, op.OperatorMatrix);
                        if (Check.InfNorm() != 0.0)
                        {
                            throw new ArgumentException("Matrix has changed, unable to re-use");
                        }
                    }
#endif
                    if (this.m_BlockingStrategy.GetNoOfBlocks(op) != this.blockSolvers.Count())
                    {
                        throw new ArgumentException("Blocking, unable to re-use");
                    }


                    return;
                }

                var Mop   = op.OperatorMatrix;
                var MgMap = op.Mapping;
                this.m_MgOp = op;
                int myMpiRank = MgMap.MpiRank;
                int myMpisize = MgMap.MpiSize;

                if (!Mop.RowPartitioning.EqualsPartition(MgMap.Partitioning))
                {
                    throw new ArgumentException("Row partitioning mismatch.");
                }
                if (!Mop.ColPartition.EqualsPartition(MgMap.Partitioning))
                {
                    throw new ArgumentException("Column partitioning mismatch.");
                }

                var ag = MgMap.AggGrid;

                int JComp  = ag.iLogicalCells.NoOfLocalUpdatedCells;
                int JGhost = ag.iLogicalCells.NoOfExternalCells;

#if DEBUG
                ilPSP.Connectors.Matlab.BatchmodeConnector matlab;
                if (m_MatlabParalellizationCheck)
                {
                    matlab = new ilPSP.Connectors.Matlab.BatchmodeConnector();
                }
                else
                {
                    matlab = null;
                }
#endif

                //Mop.Clear();
                //for(int i = Mop.RowPartitioning.i0; i < Mop.RowPartitioning.iE; i++) {
                //    Mop[i, i] = i + 1;
                //}


                // get cell blocks
                // ===============

                var _Blocks         = this.m_BlockingStrategy.GetBlocking(op);
                int NoOfSchwzBlocks = _Blocks.Count();

                // test cell blocks
                // ================
#if DEBUG
                {
                    // ensure that each cell is used exactly once, among all blocks
                    bool[] test = new bool[ag.iLogicalCells.NoOfLocalUpdatedCells];
                    foreach (var bi in _Blocks)
                    {
                        foreach (int j in bi)
                        {
                            Debug.Assert(test[j] == false);
                            test[j] = true;
                        }
                        ;
                    }
                    for (int i = 0; i < test.Length; i++)
                    {
                        Debug.Assert(test[i] == true);
                    }
                }
#endif

                // extend blocks according to desired overlap
                // ==========================================
                {
                    BitArray marker = new BitArray(JComp + JGhost);

                    if (Overlap < 0)
                    {
                        throw new ArgumentException();
                    }
                    if (Overlap > 0)
                    {
                        if (Overlap > 1 && Mop.RowPartitioning.MpiSize > 1)
                        {
                            throw new NotSupportedException("In MPI parallel runs, the maximum supported overlap for the Schwarz preconditioner is 1.");
                        }

                        foreach (List <int> bi in _Blocks) // loop over blocks...
                        {
                            marker.SetAll(false);          // marks all cells which are members of the block
                            foreach (int jcomp in bi)
                            {
                                marker[jcomp] = true;
                            }

                            // determine overlap regions
                            for (int k = 0; k < Overlap; k++)
                            {
                                int Jblock = bi.Count;
                                for (int j = 0; j < Jblock; j++)
                                {
                                    int   jCell  = bi[j];
                                    int[] Neighs = ag.iLogicalCells.CellNeighbours[jCell];
                                    foreach (int jNeigh in Neighs)
                                    {
                                        if (marker[jNeigh] == false)
                                        {
                                            // neighbor cell is not already a member of the block
                                            // => add it.
                                            bi.Add(jNeigh);
                                            marker[jNeigh] = true;
                                        }
                                    }
                                }
                            }

                            bi.Sort();
                        }
                    }

                    BlockCells = _Blocks.Select(list => list.ToArray()).ToArray();
                }


                // convert cell blocks to DOF blocks
                // =================================

                List <int>[] BlkIdx_gI_lR;                  //  for each Schwarz block, (global) indices in the local range
                List <int>[] BlkIdx_gI_eR;                  //  for each Schwarz block, (global) indices of external rows and columns
                List <int>[] TempRowIdx_gI;                 // for each Schwarz block, (global) indices into the temporary matrix
                List <int>[] BlkIdx_lI_eR;                  //  for each Schwarz block, (local)  indices of external rows and columns
                List <int>[] LocalBlocks_i0, LocalBlocks_N; // blocking of the Schwarz-Blocks.

                // for matrix 'ExternalRowsTemp': which rows of 'Mop' are required locally
                List <int> ExternalRowsIndices, ExternalRows_BlockI0, ExternalRows_BlockN;
                {
                    int Jup = MgMap.AggGrid.iLogicalCells.NoOfLocalUpdatedCells;
                    int Jgh = MgMap.AggGrid.iLogicalCells.NoOfExternalCells;

                    int LocalizedBlockCounter = 0;

                    BlkIdx_gI_lR   = NoOfSchwzBlocks.ForLoop(iPart => new List <int>(BlockCells[iPart].Length * MgMap.MaximalLength));
                    BlkIdx_gI_eR   = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
                    LocalBlocks_i0 = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
                    LocalBlocks_N  = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());

                    TempRowIdx_gI = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());
                    BlkIdx_lI_eR  = NoOfSchwzBlocks.ForLoop(iPart => new List <int>());


                    ExternalRowsIndices  = new List <int>();
                    ExternalRows_BlockI0 = new List <int>();
                    ExternalRows_BlockN  = new List <int>();

                    for (int iPart = 0; iPart < NoOfSchwzBlocks; iPart++)   // loop over parts...
                    {
                        int[] bc    = BlockCells[iPart];
                        var   biI   = BlkIdx_gI_lR[iPart];
                        var   biE   = BlkIdx_gI_eR[iPart];
                        var   l1    = TempRowIdx_gI[iPart];
                        var   l2    = BlkIdx_lI_eR[iPart];
                        var   LBBi0 = LocalBlocks_i0[iPart];
                        var   LBBN  = LocalBlocks_N[iPart];


                        int Jblock         = bc.Length;
                        int anotherCounter = 0;

                        for (int jblk = 0; jblk < Jblock; jblk++)   // loop over cells in blocks...
                        {
                            int j = bc[jblk];
                            int N = MgMap.GetLength(j);

                            if (j < Jup)
                            {
                                // locally updated cell
                                int i0 = MgMap.GlobalUniqueIndex(0, j, 0);

                                for (int n = 0; n < N; n++)
                                {
                                    biI.Add(i0 + n);
                                }
                            }
                            else
                            {
                                // external cell
                                int i0E = MgMap.GlobalUniqueIndex(0, j, 0); //
                                int i0L = MgMap.LocalUniqueIndex(0, j, 0);  //
                                ExternalRows_BlockI0.Add(LocalizedBlockCounter);
                                ExternalRows_BlockN.Add(N);
                                //LEBi0.Add(LocalizedBlockCounter);
                                //LEBn.Add(N);
                                for (int n = 0; n < N; n++)
                                {
                                    biE.Add(i0E + n);
                                    ExternalRowsIndices.Add(i0E + n);
                                    l1.Add(LocalizedBlockCounter + n);
                                    l2.Add(i0L + n);
                                    Debug.Assert(Mop._RowPartitioning.FindProcess(i0E + n) != myMpiRank);
                                }

                                LocalizedBlockCounter += N;
                            }

                            LBBi0.Add(anotherCounter);
                            LBBN.Add(N);

                            anotherCounter += N;
                        }
                    }

                    //this.BlockIndices = _LocallyStoredBlockIndices.Select(bi => bi.ToArray()).ToArray();
                }


                // get rows for blocks that use external cells
                // ===========================================

#if DEBUG
                {
                    if (Overlap == 0)
                    {
                        Debug.Assert(ExternalRowsIndices.Count == 0);
                        Debug.Assert(ExternalRows_BlockI0.Count == 0);
                        Debug.Assert(ExternalRows_BlockN.Count == 0);
                    }

                    foreach (var bi in BlkIdx_gI_lR)
                    {
                        foreach (int idx in bi)
                        {
                            Debug.Assert(idx >= m_MgOp.Mapping.i0);
                            Debug.Assert(idx < m_MgOp.Mapping.iE);
                        }
                    }

                    foreach (var ei in BlkIdx_gI_eR)
                    {
                        foreach (int idx in ei)
                        {
                            Debug.Assert(idx < m_MgOp.Mapping.i0 || idx >= m_MgOp.Mapping.iE);
                        }
                    }


                    int LL   = m_MgOp.Mapping.LocalLength;
                    int jMax = m_MgOp.Mapping.AggGrid.iLogicalCells.NoOfCells - 1;
                    int LE   = m_MgOp.Mapping.LocalUniqueIndex(0, jMax, 0) + m_MgOp.Mapping.GetLength(jMax);


                    foreach (var ci in BlkIdx_lI_eR)
                    {
                        foreach (int idx in ci)
                        {
                            Debug.Assert(idx >= LL);
                            Debug.Assert(idx < LE);
                        }
                    }

                    if (m_MatlabParalellizationCheck)
                    {
                        int globalBlockCounter = 0;
                        for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
                        {
                            int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
                            if (rankCounter == myMpiRank)
                            {
                                Debug.Assert(rank_NoBlks == NoOfSchwzBlocks);
                            }

                            for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
                            {
                                double[] vec;
                                if (rankCounter == myMpiRank)
                                {
                                    vec = ArrayTools.Cat(BlkIdx_gI_lR[iBlock], BlkIdx_gI_eR[iBlock]).Select(ii => ((double)(ii + 1))).ToArray();
                                }
                                else
                                {
                                    vec = new double[0];
                                }

                                matlab.PutVector(vec, string.Format("BlockIdx{0}", globalBlockCounter));

                                globalBlockCounter++;
                                csMPI.Raw.Barrier(csMPI.Raw._COMM.WORLD);
                            }
                        }
                    }
                }
#endif


                BlockMsrMatrix ExternalRowsTemp;
                if (myMpisize > 1 && Overlap > 0)
                {
                    //int NoOfLocalRows = _ExternalBlockIndices.Sum(L => L.Count);

                    BlockPartitioning PermRow = new BlockPartitioning(ExternalRowsIndices.Count, ExternalRows_BlockI0, ExternalRows_BlockN, Mop.MPI_Comm, i0isLocal: true);

                    // Remark: we use a permutation matrix for MPI-exchange of rows

                    BlockMsrMatrix Perm = new BlockMsrMatrix(PermRow, Mop._RowPartitioning);
                    for (int iRow = 0; iRow < ExternalRowsIndices.Count; iRow++)
                    {
                        Debug.Assert(Mop._RowPartitioning.IsInLocalRange(ExternalRowsIndices[iRow]) == false);
                        Perm[iRow + PermRow.i0, ExternalRowsIndices[iRow]] = 1;
                    }

                    ExternalRowsTemp = BlockMsrMatrix.Multiply(Perm, Mop);

#if DEBUG
                    if (m_MatlabParalellizationCheck)
                    {
                        matlab.PutSparseMatrix(Perm, "Perm");
                        matlab.PutSparseMatrix(ExternalRowsTemp, "ExternalRowsTemp");
                    }
#endif
                }
                else
                {
                    ExternalRowsTemp = null;
                }

                ExternalRowsIndices  = null;
                ExternalRows_BlockI0 = null;
                ExternalRows_BlockN  = null;



                // create solvers
                // ==============


                {
                    blockSolvers = new ISparseSolver[NoOfSchwzBlocks];

#if DEBUG
                    List <BlockMsrMatrix> Blocks = new List <BlockMsrMatrix>();
#endif
                    for (int iPart = 0; iPart < NoOfSchwzBlocks; iPart++)
                    {
                        var bi = BlkIdx_gI_lR[iPart];

                        int Bsz;
                        if (MgMap.MinimalLength == MgMap.MaximalLength)
                        {
                            Bsz = MgMap.MaximalLength;
                        }
                        else
                        {
                            Bsz = 1;
                        }

                        var l1 = TempRowIdx_gI[iPart];

                        //if (M.RowPartitioning.MpiSize > 1) {
                        //    int i0Proc = M.RowPartitioning.i0;
                        //    bi = bi.CloneAs();
                        //    for (int i = 0; i < bi.Length; i++) {
                        //        bi[i] += i0Proc;
                        //    }
                        //}

                        BlockPartitioning localBlocking = new BlockPartitioning(bi.Count + l1.Count, LocalBlocks_i0[iPart], LocalBlocks_N[iPart], csMPI.Raw._COMM.SELF);

                        if (l1.Count > 0)
                        {
                            // convert the indices into 'ExternalRowsTemp' to global indices
                            int l1L    = l1.Count;
                            int offset = ExternalRowsTemp._RowPartitioning.i0;
                            for (int i = 0; i < l1L; i++)
                            {
                                l1[i] += offset;
                            }
                        }

                        BlockMsrMatrix Block = new BlockMsrMatrix(localBlocking, localBlocking);// bi.Length, bi.Length, Bsz, Bsz);
                        Mop.WriteSubMatrixTo(Block, bi, default(int[]), bi, default(int[]));
                        if (l1.Count > 0)
                        {
                            int   offset   = bi.Count;
                            int[] targRows = l1.Count.ForLoop(i => i + offset);

                            var   biE         = BlkIdx_gI_eR[iPart];
                            int[] extTargCols = biE.Count.ForLoop(i => i + offset);

                            Mop.AccSubMatrixTo(1.0, Block, bi, default(int[]), new int[0], default(int[]), biE, extTargCols);
                            ExternalRowsTemp.AccSubMatrixTo(1.0, Block, l1, targRows, bi, default(int[]), biE, extTargCols);
                        }
#if DEBUG
                        if (m_MatlabParalellizationCheck)
                        {
                            Blocks.Add(Block);
                        }
#endif
                        blockSolvers[iPart] = new PARDISOSolver()
                        {
                            CacheFactorization = true,
                            UseDoublePrecision = false
                        };
                        //blockSolvers[iPart] = new FullDirectSolver();
                        //blockSolvers[iPart] = new ilPSP.LinSolvers.MUMPS.MUMPSSolver();
                        blockSolvers[iPart].DefineMatrix(Block);
                    }

#if DEBUG
                    if (m_MatlabParalellizationCheck)
                    {
                        int globalBlockCounter = 0;
                        for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
                        {
                            int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
                            for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
                            {
                                BlockMsrMatrix Block;
                                if (rankCounter == myMpiRank)
                                {
                                    Block = Blocks[iBlock];
                                }
                                else
                                {
                                    Block = null;
                                }

                                matlab.PutSparseMatrix(Block, string.Format("Block{0}", globalBlockCounter));

                                globalBlockCounter++;
                                csMPI.Raw.Barrier(csMPI.Raw._COMM.WORLD);
                            }
                        }
                    }
#endif
                }

                // Record required indices
                // =======================
                {
                    this.BlockIndices_Local    = new int[NoOfSchwzBlocks][];
                    this.BlockIndices_External = new int[NoOfSchwzBlocks][];
                    int LocalI0     = MgMap.i0;
                    int LocalLength = MgMap.LocalLength;

                    for (int iBlock = 0; iBlock < NoOfSchwzBlocks; iBlock++)
                    {
                        var   _bi = BlkIdx_gI_lR[iBlock];
                        int   L   = _bi.Count;
                        int[] bil = new int[L];
                        this.BlockIndices_Local[iBlock] = bil;

                        for (int l = 0; l < L; l++)
                        {
                            bil[l] = _bi[l] - LocalI0;
                            Debug.Assert(bil[l] >= 0);
                            Debug.Assert(bil[l] < MgMap.LocalLength);
                        }

                        var _biE = BlkIdx_lI_eR[iBlock];
                        if (_biE.Count > 0)
                        {
                            this.BlockIndices_External[iBlock] = _biE.ToArray();
                        }
                    }
                }


                this.MtxFull = Mop;

                if (CoarseSolver != null)
                {
                    CoarseSolver.Init(op.CoarserLevel);
                }

                // Debug & Test-Code
                // =================
#if DEBUG
                if (m_MatlabParalellizationCheck)
                {
                    Console.WriteLine("Matlab dir: " + matlab.WorkingDirectory);

                    matlab.PutSparseMatrix(Mop, "Full");
                    int GlobalNoOfBlocks = NoOfSchwzBlocks.MPISum();



                    for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
                    {
                        matlab.Cmd("BlockErr({0} + 1, 1) = norm( Block{0} - Full( BlockIdx{0}, BlockIdx{0} ), inf );", iGlbBlock);
                    }

                    Random   rnd     = new Random(myMpiRank);
                    double[] testRHS = new double[MgMap.LocalLength];
                    for (int i = 0; i < testRHS.Length; i++)
                    {
                        testRHS[i] = rnd.NextDouble();
                    }
                    matlab.PutVector(testRHS, "testRHS");

                    MPIexchange <double[]> ResExchange = new MPIexchange <double[]>(MgMap, testRHS);
                    ResExchange.TransceiveStartImReturn();
                    ResExchange.TransceiveFinish(0.0);

                    int offset = MgMap.LocalLength;

                    int g = 0;
                    for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
                    {
                        int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
                        for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
                        {
                            double[] SubVec;
                            if (rankCounter == myMpiRank)
                            {
                                int LL = this.BlockIndices_Local[iBlock].Length;
                                int LE;
                                if (this.BlockIndices_External[iBlock] != null)
                                {
                                    LE = this.BlockIndices_External[iBlock].Length;
                                }
                                else
                                {
                                    LE = 0;
                                }
                                int L = LL + LE;

                                SubVec = new double[L];
                                for (int i = 0; i < LL; i++)
                                {
                                    SubVec[i] = testRHS[this.BlockIndices_Local[iBlock][i]];
                                }
                                if (LE > 0)
                                {
                                    for (int i = 0; i < LE; i++)
                                    {
                                        SubVec[i + LL] = ResExchange.Vector_Ext[this.BlockIndices_External[iBlock][i] - offset];
                                    }
                                }
                            }
                            else
                            {
                                SubVec = new double[0];
                            }

                            matlab.PutVector(SubVec, "SubVec" + g);

                            g++;
                        }
                    }

                    for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
                    {
                        matlab.Cmd("RhsErr({0} + 1, 1) = norm( SubVec{0} - testRHS( BlockIdx{0} ), inf );", iGlbBlock);
                    }

                    double[] testX = new double[testRHS.Length];
                    MPIexchangeInverse <double[]> XExchange = new MPIexchangeInverse <double[]>(MgMap, testX);

                    g = 0;
                    for (int rankCounter = 0; rankCounter < myMpisize; rankCounter++)
                    {
                        int rank_NoBlks = NoOfSchwzBlocks.MPIBroadcast(rankCounter);
                        for (int iBlock = 0; iBlock < rank_NoBlks; iBlock++)
                        {
                            if (rankCounter == myMpiRank)
                            {
                                int LL = this.BlockIndices_Local[iBlock].Length;
                                int LE;
                                if (this.BlockIndices_External[iBlock] != null)
                                {
                                    LE = this.BlockIndices_External[iBlock].Length;
                                }
                                else
                                {
                                    LE = 0;
                                }
                                int L = LL + LE;


                                for (int i = 0; i < LL; i++)
                                {
                                    testX[this.BlockIndices_Local[iBlock][i]] += (g + 1);
                                }
                                if (LE > 0)
                                {
                                    for (int i = 0; i < LE; i++)
                                    {
                                        XExchange.Vector_Ext[this.BlockIndices_External[iBlock][i] - offset] += (g + 1);
                                    }
                                }
                            }
                            else
                            {
                                //nop
                            }

                            g++;
                        }
                    }
                    XExchange.TransceiveStartImReturn();
                    XExchange.TransceiveFinish(1.0);

                    matlab.Cmd("testXref = zeros({0},1);", MgMap.TotalLength);
                    for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
                    {
                        matlab.Cmd("testXref(BlockIdx{0},1) = testXref(BlockIdx{0},1) + ({0} + 1);", iGlbBlock);
                    }

                    matlab.PutVector(testX, "testX");
                    matlab.Cmd("testXErr = norm(testX - testXref, inf);");

                    MultidimensionalArray BlockErr = MultidimensionalArray.Create(GlobalNoOfBlocks, 1);
                    MultidimensionalArray RhsErr   = MultidimensionalArray.Create(GlobalNoOfBlocks, 1);
                    MultidimensionalArray testXErr = MultidimensionalArray.Create(1, 1);

                    matlab.GetMatrix(BlockErr, "BlockErr");
                    matlab.GetMatrix(RhsErr, "RhsErr");
                    matlab.GetMatrix(testXErr, "testXErr");

                    matlab.Execute();

                    for (int iGlbBlock = 0; iGlbBlock < GlobalNoOfBlocks; iGlbBlock++)
                    {
                        Console.WriteLine("Block #{0} Error (external? ) " + BlockErr[iGlbBlock, 0], iGlbBlock);
                        Console.WriteLine("RHS #{0} Error " + RhsErr[iGlbBlock, 0], iGlbBlock);
                        Debug.Assert(BlockErr[iGlbBlock, 0] == 0);
                        Debug.Assert(RhsErr[iGlbBlock, 0] == 0);
                    }

                    Console.WriteLine("X Error " + testXErr[0, 0]);
                    Debug.Assert(testXErr[0, 0] == 0.0);

                    matlab.Dispose();
                }
#endif
            }
        }
Exemplo n.º 7
0
        /// <summary>
        /// ~
        /// </summary>
        public void Init(MultigridOperator op)
        {
            //System.Threading.Thread.Sleep(10000);
            //ilPSP.Environment.StdoutOnlyOnRank0 = false;
            m_op = op;

            var Map    = op.Mapping;
            int NoVars = Map.AggBasis.Length;
            int j0     = Map.FirstBlock;
            int J      = Map.LocalNoOfBlocks;

            int[] degs = m_op.Degrees;

            for (int ideg = 0; ideg < degs.Length; ideg++)
            {
                if (degs[ideg] == 0)
                {
                    throw new ArgumentException(String.Format("DGdegree for Variable {0} is 0, p-multigrid not possible", ideg));
                }
            }

            var BS = Map.AggBasis;

            if (UseHiOrderSmoothing)
            {
                HighOrderBlocks_LU       = new MultidimensionalArray[J];
                HighOrderBlocks_LUpivots = new int[J][];
                HighOrderBlocks_indices  = new int[J][];
            }


            var GsubIdx           = new List <int>();
            var LsubIdx           = new List <int>();
            var lowLocalBlocks_i0 = new List <int>();
            var lowLocalBlocks__N = new List <int>();
            int cnt = 0;

            /*
             * var debugerSW = new StreamWriter(String.Concat("debug_of_", ilPSP.Environment.MPIEnv.MPI_Rank));
             * debugerSW.WriteLine("proc {0} reporting ...",ilPSP.Environment.MPIEnv.MPI_Rank);
             * debugerSW.WriteLine("Num of Blocks {0}",HighOrderBlocks_LUpivots.Length);
             */

            for (int jLoc = 0; jLoc < J; jLoc++)
            {
                lowLocalBlocks_i0.Add(cnt);
                lowLocalBlocks__N.Add(0);


                var LhiIdx             = new List <int>();
                var IdxHighBlockOffset = new int[NoVars][];
                var IdxHighOffset      = new int[NoVars][];

                int NpHiTot = 0;
                for (int iVar = 0; iVar < NoVars; iVar++)
                {
                    int pReq;
                    if (degs[iVar] <= 1)
                    {
                        pReq = 0;
                    }
                    else
                    {
                        pReq = CoarseLowOrder;
                    }

                    int Np1 = BS[iVar].GetLength(jLoc, pReq);
                    int Np  = BS[iVar].GetLength(jLoc, degs[iVar]);
                    lowLocalBlocks__N[jLoc] += Np1;

                    int NoOfSpc   = BS[iVar].GetNoOfSpecies(jLoc);
                    int NpBase    = Np / NoOfSpc;
                    int NpBaseLow = Np1 / NoOfSpc;
                    IdxHighBlockOffset[iVar] = new int[NoOfSpc + 1];
                    IdxHighOffset[iVar]      = new int[NoOfSpc];

                    for (int iSpc = 0; iSpc < NoOfSpc; iSpc++)
                    {
                        int n          = 0;
                        int cellOffset = NpBase * iSpc;
                        IdxHighOffset[iVar][iSpc] = Map.GlobalUniqueIndex(iVar, jLoc, cellOffset + NpBaseLow);

                        for (; n < NpBaseLow; n++)
                        {
                            int Lidx = Map.LocalUniqueIndex(iVar, jLoc, n + cellOffset);
                            LsubIdx.Add(Lidx); //local block mapping Coarse Matrix (low order entries) to original matrix

                            int Gidx = Map.GlobalUniqueIndex(iVar, jLoc, n + cellOffset);
                            GsubIdx.Add(Gidx); //global mapping Coarse Matrix (low order entries) to original matrix
                        }

                        for (; n < NpBase; n++)
                        {
                            int Lidx = Map.LocalUniqueIndex(iVar, jLoc, n + cellOffset);
                            LhiIdx.Add(Lidx); //local block mapping of high order entries to original matrix

                            //int Gidx = Map.GlobalUniqueIndex(iVar, jLoc, n);
                            //GhiIdx.Add(Gidx);
                        }

                        IdxHighBlockOffset[iVar][iSpc] = NpHiTot;
                        NpHiTot += (NpBase - NpBaseLow);
                    }
                    IdxHighBlockOffset[iVar][NoOfSpc] = NpHiTot;

                    Debug.Assert(GsubIdx.Last() == Map.GlobalUniqueIndex(iVar, jLoc, (NoOfSpc - 1) * NpBase + NpBaseLow - 1));
                    Debug.Assert(LhiIdx.Last() == Map.LocalUniqueIndex(iVar, jLoc, NoOfSpc * NpBase - 1));
                }

                // Save high order blocks for later smoothing
                if (NpHiTot > 0)
                {
                    HighOrderBlocks_LU[jLoc]      = MultidimensionalArray.Create(NpHiTot, NpHiTot);
                    HighOrderBlocks_indices[jLoc] = LhiIdx.ToArray();

                    for (int iVar = 0; iVar < NoVars; iVar++)
                    {
                        for (int jVar = 0; jVar < NoVars; jVar++)
                        {
                            for (int iSpc = 0; iSpc < BS[jVar].GetNoOfSpecies(jLoc); iSpc++)
                            {
                                int i0_hi  = IdxHighOffset[iVar][iSpc];
                                int j0_hi  = IdxHighOffset[jVar][iSpc];
                                int Row_i0 = IdxHighBlockOffset[iVar][iSpc];
                                int Col_i0 = IdxHighBlockOffset[jVar][iSpc];
                                int Row_ie = IdxHighBlockOffset[iVar][iSpc + 1];
                                int col_ie = IdxHighBlockOffset[jVar][iSpc + 1];

                                //debugerSW.WriteLine("Block {0}: i0={1} j0={2} iVar={3}", jLoc, i0_hi, j0_hi, iVar);

                                m_op.OperatorMatrix.ReadBlock(i0_hi, j0_hi,
                                                              HighOrderBlocks_LU[jLoc].ExtractSubArrayShallow(new int[] { Row_i0, Col_i0 }, new int[] { Row_ie - 1, col_ie - 1 }));
                            }
                        }
                    }

                    HighOrderBlocks_LUpivots[jLoc] = new int[NpHiTot];
                    HighOrderBlocks_LU[jLoc].FactorizeLU(HighOrderBlocks_LUpivots[jLoc]);
                }

                cnt += lowLocalBlocks__N[jLoc];
            }



            m_LsubIdx = LsubIdx.ToArray();

            BlockPartitioning localBlocking = new BlockPartitioning(GsubIdx.Count, lowLocalBlocks_i0, lowLocalBlocks__N, Map.MPI_Comm, i0isLocal: true);
            var P01SubMatrix = new BlockMsrMatrix(localBlocking);

            op.OperatorMatrix.AccSubMatrixTo(1.0, P01SubMatrix, GsubIdx, default(int[]), GsubIdx, default(int[]));

            intSolver = new PARDISOSolver()
            {
                CacheFactorization = true,
                UseDoublePrecision = false
            };
            intSolver.DefineMatrix(P01SubMatrix);

            /*
             * LsubIdx.SaveToTextFileDebug("LsubIdx");
             * GsubIdx.SaveToTextFileDebug("GsubIdx");
             * P01SubMatrix.SaveToTextFileSparseDebug("lowM");
             * m_op.OperatorMatrix.SaveToTextFileSparseDebug("M");
             * P01SubMatrix.SaveToTextFileSparse("lowM_full");
             * m_op.OperatorMatrix.SaveToTextFileSparse("M_full");
             *
             * debugerSW.WriteLine("Dim of lowMatrix: {0}",GsubIdx.Count);
             * debugerSW.Flush();
             * debugerSW.Close();
             */
        }