Example #1
0
        public void Solve <U, V>(U X, V B)
            where U : IList <double>
            where V : IList <double>
        {
            //using (var solver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver()) {
            //    solver.DefineMatrix(P);
            //    solver.Solve(X, B);
            //}

            switch (SchurOpt)
            {
            case SchurOptions.exact:
            {
                // Directly invert Preconditioning Matrix
                using (var solver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver())
                {
                    solver.DefineMatrix(P);
                    solver.Solve(X, B);
                }
                return;
            }

            case SchurOptions.decoupledApprox:
            {
                SolveSubproblems(X, B);
                return;
            }

            case SchurOptions.SIMPLE:
            {
                SolveSIMPLE(X, B);
                return;
            }
            }
        }
Example #2
0
        /// <summary>
        /// Solve with a SIMPLE like approcation of the Schur complement
        /// </summary>
        /// <typeparam name="U"></typeparam>
        /// <typeparam name="V"></typeparam>
        /// <param name="X"></param>
        /// <param name="B"></param>
        public void SolveSIMPLE <U, V>(U X, V B)
            where U : IList <double>
            where V : IList <double>
        {
            var Bu = new double[Uidx.Length];
            var Xu = Bu.CloneAs();

            // For MPI
            var idxU = Uidx[0];

            for (int i = 0; i < Uidx.Length; i++)
            {
                Uidx[i] -= idxU;
            }
            var idxP = Pidx[0];

            for (int i = 0; i < Pidx.Length; i++)
            {
                Pidx[i] -= idxP;
            }

            Bu = B.GetSubVector(Uidx, default(int[]));
            var Bp = new double[Pidx.Length];
            var Xp = Bp.CloneAs();

            Bp = B.GetSubVector(Pidx, default(int[]));

            Xu = X.GetSubVector(Uidx, default(int[]));
            Xp = X.GetSubVector(Pidx, default(int[]));

            // Solve SIMPLE Schur
            using (var solver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver())
            {
                solver.DefineMatrix(simpleSchur);
                solver.Solve(Xp, Bp);
            }


            // Solve ConvDiff*w=v-q*pGrad
            pGrad.SpMVpara(-1, Xp, 1, Bu);
            using (var solver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver())
            {
                solver.DefineMatrix(ConvDiff);
                solver.Solve(Xu, Bu);
            }

            var temp = new double[Uidx.Length + Pidx.Length];

            for (int i = 0; i < Uidx.Length; i++)
            {
                temp[Uidx[i]] = Xu[i];
            }

            for (int i = 0; i < Pidx.Length; i++)
            {
                temp[Pidx[i]] = Xp[i];
            }

            X.SetV(temp);
        }
Example #3
0
 public void Solve <U, V>(U X, V B)
     where U : IList <double>
     where V : IList <double>
 {
     using (var solver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver())
     {
         solver.DefineMatrix(P);
         solver.Solve(X, B);
     }
     m_ThisLevelIterations++;
 }
Example #4
0
 /// <summary>
 /// Solves a linear system using a direct solver.
 /// </summary>
 /// <param name="Matrix">the matrix of the linear problem.</param>
 /// <param name="X">Output, (hopefully) the solution.</param>
 /// <param name="B">Input, the right-hand-side.</param>
 /// <returns>Actual number of iterations.</returns>
 static public void Solve_Direct <V, W>(this IMutableMatrixEx Matrix, V X, W B)
     where V : IList <double>
     where W : IList <double> //
 {
     /*
      * using (var slv = new ilPSP.LinSolvers.PARDISO.PARDISOSolver()) {
      *  slv.DefineMatrix(Matrix);
      *  var SolRes = slv.Solve(X, B.ToArray());
      * }
      */
     using (var slv = new ilPSP.LinSolvers.MUMPS.MUMPSSolver()) {
         slv.DefineMatrix(Matrix);
         var SolRes = slv.Solve(X, B.ToArray());
     }
 }
Example #5
0
        /// <summary>
        /// Condition number estimate by MUMPS; Rem.: MUMPS manual does not tell in which norm; it does not seem to be as reliable as MATLAB condest
        /// </summary>
        public static double Condest_MUMPS(this IMutableMatrixEx Mtx)
        {
            using (new FuncTrace()) {
                using (var slv = new ilPSP.LinSolvers.MUMPS.MUMPSSolver()) {
                    slv.Statistics = MUMPS.MUMPSStatistics.AllStatistics;

                    slv.DefineMatrix(Mtx);
                    double[] dummyRHS = new double[Mtx.RowPartitioning.LocalLength];
                    double[] dummySol = new double[dummyRHS.Length];
                    dummyRHS.SetAll(1.11);

                    slv.Solve(dummySol, dummyRHS);

                    return(slv.LastCondNo);
                }
            }
        }
Example #6
0
        /// <summary>
        /// Approximate the inverse of the Schur matrix and perform two Poisson solves and Matrix-Vector products. Finite elements and Fast Iterative Solvers p.383
        /// </summary>
        public void ApproxAndSolveSchur <U, V>(U Xp, V Bp)
            where U : IList <double>
            where V : IList <double>
        {
            var temp = new double[Xp.Count];
            var sol  = new double[pGrad.RowPartitioning.LocalLength];

            // Poisson solve
            using (var solver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver())
            {
                solver.DefineMatrix(PoissonMtx_T);
                solver.Solve(temp, Bp);
            }

            // Schur Convective part with scaling
            pGrad.SpMVpara(1, temp, 0, sol);

            temp = sol.ToArray();
            sol.Clear();
            invVelMassMatrix.SpMVpara(1, temp, 0, sol);

            temp = sol.ToArray();
            sol.Clear();
            ConvDiff.SpMVpara(1, temp, 0, sol);

            temp = sol.ToArray();
            sol.Clear();
            invVelMassMatrixSqrt.SpMVpara(1, temp, 0, sol);

            temp = sol.ToArray();
            divVel.SpMVpara(1, temp, 0, Xp);

            // Poisson solve
            using (var solver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver())
            {
                solver.DefineMatrix(PoissonMtx_H);
                solver.Solve(Xp, Xp);
            }
        }
Example #7
0
        /// <summary>
        /// Solution of the system
        /// <see cref="LaplaceMtx"/>*<see cref="T"/> + <see cref="LaplaceAffine"/> = <see cref="RHS"/>
        /// using a black-box solver
        /// </summary>
        private void ClassicSolve(out double mintime, out double maxtime, out bool Converged, out int NoOfIter)
        {
            mintime   = double.MaxValue;
            maxtime   = double.MinValue;
            Converged = false;
            NoOfIter  = int.MaxValue;

            for (int i = 0; i < base.Control.NoOfSolverRuns; i++)
            {
                // create sparse solver
                // --------------------
                ISparseSolver    ipSolver;
                LinearSolverCode solvercodes = LinearSolverCode.classic_pardiso;

                switch (solvercodes)
                {
                case LinearSolverCode.classic_pardiso:
                    ipSolver = new ilPSP.LinSolvers.PARDISO.PARDISOSolver()
                    {
                        CacheFactorization = true,
                        UseDoublePrecision = true
                    };
                    break;

                case LinearSolverCode.classic_mumps:
                    ipSolver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver();
                    break;

                case LinearSolverCode.classic_cg:
                    ipSolver = new ilPSP.LinSolvers.monkey.CG()
                    {
                        MaxIterations = 1000000,
                        Tolerance     = 1.0e-10,
                        DevType       = ilPSP.LinSolvers.monkey.DeviceType.Cuda
                    };
                    break;

                default:
                    throw new ArgumentException();
                }
                ipSolver.DefineMatrix(LaplaceMtx);

                // call solver
                // -----------
                T.Clear();

                Console.WriteLine("RUN " + i + ": solving system...");

                var RHSvec = RHS.CoordinateVector.ToArray();
                BLAS.daxpy(RHSvec.Length, -1.0, this.LaplaceAffine, 1, RHSvec, 1);

                T.Clear();
                SolverResult solRes = ipSolver.Solve(T.CoordinateVector, RHSvec);
                mintime = Math.Min(solRes.RunTime.TotalSeconds, mintime);
                maxtime = Math.Max(solRes.RunTime.TotalSeconds, maxtime);

                Converged = solRes.Converged;
                NoOfIter  = solRes.NoOfIterations;

                Console.WriteLine("Pardiso phase 11: " + ilPSP.LinSolvers.PARDISO.PARDISOSolver.Phase_11.Elapsed.TotalSeconds);
                Console.WriteLine("Pardiso phase 22: " + ilPSP.LinSolvers.PARDISO.PARDISOSolver.Phase_22.Elapsed.TotalSeconds);
                Console.WriteLine("Pardiso phase 33: " + ilPSP.LinSolvers.PARDISO.PARDISOSolver.Phase_33.Elapsed.TotalSeconds);

                ipSolver.Dispose();
            }
        }
        public void Solve <U, V>(U X, V B)
            where U : IList <double>
            where V : IList <double>
        {
            var Bu = new double[Uidx.Length];
            var Xu = Bu.CloneAs();

            Bu = B.GetSubVector(Uidx, default(int[]));
            var Bp = new double[Pidx.Length];
            var Xp = Bp.CloneAs();

            Bp = B.GetSubVector(Pidx, default(int[]));

            Xu = X.GetSubVector(Uidx, default(int[]));
            Xp = X.GetSubVector(Pidx, default(int[]));

            P = new BlockMsrMatrix(null, null);// Pidx.Length, Pidx.Length); //

            MultidimensionalArray Schur = MultidimensionalArray.Create(Pidx.Length, Pidx.Length);

            using (BatchmodeConnector bmc = new BatchmodeConnector()) {
                bmc.PutSparseMatrix(LocalMatrix, "LocalMatrix");
                bmc.PutSparseMatrix(divVel, "divVel");
                bmc.PutSparseMatrix(pGrad, "pGrad");
                bmc.Cmd("invLocalMatrix = inv(full(LocalMatrix))");
                bmc.Cmd("Poisson = invLocalMatrix*pGrad");
                bmc.Cmd("Schur= divVel*Poisson");
                bmc.GetMatrix(Schur, "Schur");
                bmc.Execute(false);
            }

            P = null; // Schur.ToMsrMatrix();

            //P.SaveToTextFileSparse("LocalSchur");

            using (var solver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver()) {
                solver.DefineMatrix(P);
                solver.Solve(Xp, Bp);
            }

            //var temp2 = Xp.CloneAs();

            //LocalMatrix.SpMV(1, Xp, 0, temp2);

            //using (var solver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver())
            //{
            //    solver.DefineMatrix(divVel);
            //    solver.Solve(Xp, temp2);
            //}


            // Solve ConvD           iff*w=v-q*pGrad
            pGrad.SpMV(-1, Xp, 1, Bu);

            using (var solver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver()) {
                solver.DefineMatrix(ConvDiff);
                solver.Solve(Xu, Bu);
            }

            var temp = new double[Uidx.Length + Pidx.Length];

            for (int i = 0; i < Uidx.Length; i++)
            {
                temp[Uidx[i]] = Xu[i];
            }

            for (int i = 0; i < Pidx.Length; i++)
            {
                temp[Pidx[i]] = Xp[i];
            }

            X.SetV(temp);
            m_ThisLevelIterations++;
        }
Example #9
0
        public override void SolverDriver <S>(CoordinateVector SolutionVec, S RHS)
        {
            using (var tr = new FuncTrace()) {
                m_SolutionVec = SolutionVec;

                int itc;
                itc = 0;
                double[] x, xt, xOld, f0, deltaX, ft;
                double   rat;
                double   alpha = 1E-4, sigma0 = 0.1, sigma1 = 0.5, maxarm = 20, gamma = 0.9;

                // Eval_F0
                base.Init(SolutionVec, RHS, out x, out f0);

                Console.WriteLine("Residual base.init:   " + f0.L2NormPow2().MPISum().Sqrt());


                deltaX = new double[x.Length];
                xt     = new double[x.Length];
                ft     = new double[x.Length];


                this.CurrentLin.TransformSolFrom(SolutionVec, x);
                base.EvalResidual(x, ref f0);


                // fnorm
                double    fnorm  = f0.L2NormPow2().MPISum().Sqrt();
                double    fNormo = 1;
                double    errstep;
                double[]  step    = new double[x.Length];
                double[]  stepOld = new double[x.Length];
                MsrMatrix CurrentJac;

                Console.WriteLine("Start residuum for nonlinear iteration:  " + fnorm);

                OnIterationCallback(itc, x.CloneAs(), f0.CloneAs(), this.CurrentLin);

                while (fnorm > ConvCrit && itc < MaxIter)
                {
                    rat    = fnorm / fNormo;
                    fNormo = fnorm;
                    itc++;

                    // How should the inverse of the Jacobian be approximated?
                    if (ApproxJac == ApproxInvJacobianOptions.GMRES)
                    {
                        CurrentJac = new MsrMatrix(x.Length);
                        if (Precond != null)
                        {
                            Precond.Init(CurrentLin);
                        }
                        step = Krylov(SolutionVec, x, f0, out errstep);
                    }
                    else if (ApproxJac == ApproxInvJacobianOptions.DirectSolver)
                    {
                        CurrentJac = diffjac(SolutionVec, x, f0);
                        var solver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver();
                        solver.DefineMatrix(CurrentJac);
                        step.ClearEntries();
                        solver.Solve(step, f0);
                    }
                    else
                    {
                        throw new NotImplementedException("Your approximation option for the jacobian seems not to be existent.");
                    }

                    // Start line search
                    xOld = x;
                    double lambda = 1;
                    double lamm   = 1;
                    double lamc   = lambda;
                    double iarm   = 0;
                    xt = x.CloneAs();
                    xt.AccV(lambda, step);
                    this.CurrentLin.TransformSolFrom(SolutionVec, xt);
                    EvaluateOperator(1, SolutionVec.Mapping.Fields, ft);
                    var nft = ft.L2NormPow2().MPISum().Sqrt(); var nf0 = f0.L2NormPow2().MPISum().Sqrt(); var ff0 = nf0 * nf0; var ffc = nft * nft; var ffm = nft * nft;

                    // Control of the the step size
                    while (nft >= (1 - alpha * lambda) * nf0 && iarm < maxStep)
                    {
                        // Line search starts here

                        if (iarm == 0)
                        {
                            lambda = sigma1 * lambda;
                        }
                        else
                        {
                            lambda = parab3p(lamc, lamm, ff0, ffc, ffm);
                        }

                        // Update x;
                        xt = x.CloneAs();
                        xt.AccV(lambda, step);
                        lamm = lamc;
                        lamc = lambda;

                        this.CurrentLin.TransformSolFrom(SolutionVec, xt);

                        EvaluateOperator(1, SolutionVec.Mapping.Fields, ft);

                        nft = ft.L2NormPow2().MPISum().Sqrt();
                        ffm = ffc;
                        ffc = nft * nft;
                        iarm++;

#if DEBUG
                        Console.WriteLine("Step size:  " + lambda + "with Residuum:  " + nft);
#endif
                    }
                    // transform solution back to 'original domain'
                    // to perform the linearization at the new point...
                    // (and for Level-Set-Updates ...)
                    this.CurrentLin.TransformSolFrom(SolutionVec, xt);

                    // update linearization
                    base.Update(SolutionVec.Mapping.Fields, ref xt);

                    // residual evaluation & callback
                    base.EvalResidual(xt, ref ft);

                    // EvaluateOperator(1, SolutionVec.Mapping.Fields, ft);

                    //base.Init(SolutionVec, RHS, out x, out f0);

                    fnorm = ft.L2NormPow2().MPISum().Sqrt();

                    x  = xt;
                    f0 = ft.CloneAs();

                    OnIterationCallback(itc, x.CloneAs(), f0.CloneAs(), this.CurrentLin);
                }

                SolutionVec = m_SolutionVec;
            }
        }
Example #10
0
        public void Init(MultigridOperator op)
        {
            var M     = op.OperatorMatrix;
            var MgMap = op.Mapping;

            this.m_MgOp = op;

            if (!M.RowPartitioning.EqualsPartition(MgMap.Partitioning))
            {
                throw new ArgumentException("Row partitioning mismatch.");
            }
            if (!M.ColPartition.EqualsPartition(MgMap.Partitioning))
            {
                throw new ArgumentException("Column partitioning mismatch.");
            }

            var ag = MgMap.AggGrid;

            int JComp  = ag.iLogicalCells.NoOfLocalUpdatedCells;
            int JGhost = ag.iLogicalCells.NoOfExternalCells;

            // get cell blocks
            // ===============

            var _Blocks = this.m_BlockingStrategy.GetBlocking(op);
            int NoParts = _Blocks.Count();

            // test cell blocks
            // ================
#if DEBUG
            {
                // ensure that each cell is used exactly once, among all blocks
                bool[] test = new bool[ag.iLogicalCells.NoOfLocalUpdatedCells];
                foreach (var bi in _Blocks)
                {
                    foreach (int j in bi)
                    {
                        Debug.Assert(test[j] == false);
                        test[j] = true;
                    }
                    ;
                }
                for (int i = 0; i < test.Length; i++)
                {
                    Debug.Assert(test[i] == true);
                }
            }
#endif

            // extend blocks according to desired overlap
            // ==========================================
            {
                BitArray marker = new BitArray(JComp + JGhost);

                if (overlap < 0)
                {
                    throw new ArgumentException();
                }
                if (overlap > 0)
                {
                    foreach (List <int> bi in _Blocks) // loop over blocks...
                    {
                        marker.SetAll(false);          // marks all cells which are members of the block
                        foreach (int jcomp in bi)
                        {
                            marker[jcomp] = true;
                        }

                        // determine overlap regions
                        for (int k = 0; k < overlap; k++)
                        {
                            int Jblock = bi.Count;
                            for (int j = 0; j < Jblock; j++)
                            {
                                int   jCell  = bi[j];
                                int[] Neighs = ag.iLogicalCells.CellNeighbours[jCell];
                                foreach (int jNeigh in Neighs)
                                {
                                    if (marker[jNeigh] == false)
                                    {
                                        // neighbor cell is not already a member of the block
                                        // => add it.
                                        bi.Add(jNeigh);
                                        marker[jNeigh] = true;
                                    }
                                }
                            }
                        }

                        bi.Sort();
                    }
                }


                BlockCells = _Blocks.Select(list => list.ToArray()).ToArray();
            }


            // convert cell blocks to DOF blocks
            // =================================

            {
                int Jup = MgMap.AggGrid.iLogicalCells.NoOfLocalUpdatedCells;
                int Jgh = MgMap.AggGrid.iLogicalCells.NoOfExternalCells;


                var _BlockIndices = NoParts.ForLoop(iPart => new List <int>(BlockCells[iPart].Length * MgMap.MaximalLength));
                for (int iPart = 0; iPart < NoParts; iPart++)   // loop over parts...
                {
                    int[] bc     = BlockCells[iPart];
                    var   bi     = _BlockIndices[iPart];
                    int   Jblock = bc.Length;


                    for (int jblk = 0; jblk < Jblock; jblk++)
                    {
                        int j = bc[jblk];

                        if (j < Jup)
                        {
                            int N  = MgMap.GetLength(j);
                            int i0 = MgMap.LocalUniqueIndex(0, j, 0);

                            for (int n = 0; n < N; n++)
                            {
                                bi.Add(i0 + n);
                            }
                        }
                        else
                        {
                            throw new NotImplementedException("todo: MPI parallelization;");
                        }
                    }
                }

                this.BlockIndices = _BlockIndices.Select(bi => bi.ToArray()).ToArray();
            }


            // test DOF blocks
            // ===============
            {
                int L  = M.RowPartitioning.LocalLength;
                int i0 = M.RowPartitioning.i0;

                // ensure that each cell is used exactly once, among all blocks

                BitArray test = new BitArray(L);
                foreach (var bi in BlockIndices)
                {
                    bi.ForEach(delegate(int i) {
                        if (i < L)
                        {
                            Debug.Assert(test[i] == false || this.overlap > 0);
                            test[i] = true;
                        }
                    });
                }
                for (int i = 0; i < L; i++)
                {
                    Debug.Assert(test[i] == true);
                }
            }


            // create solvers
            // ==============

            {
                blockSolvers = new ISparseSolver[NoParts];

                for (int iPart = 0; iPart < NoParts; iPart++)
                {
                    int[] bi = BlockIndices[iPart];

                    int Bsz;
                    if (MgMap.MinimalLength == MgMap.MaximalLength)
                    {
                        Bsz = MgMap.MaximalLength;
                    }
                    else
                    {
                        Bsz = 1;
                    }

                    if (M.RowPartitioning.MpiSize > 1)
                    {
                        int i0Proc = M.RowPartitioning.i0;
                        bi = bi.CloneAs();
                        for (int i = 0; i < bi.Length; i++)
                        {
                            bi[i] += i0Proc;
                        }
                    }


                    MsrMatrix Block = new MsrMatrix(bi.Length, bi.Length, Bsz, Bsz);
                    M.WriteSubMatrixTo(Block, bi, default(int[]), bi, default(int[]));

                    //blockSolvers[iPart] = new PARDISOSolver();
                    //blockSolvers[iPart].CacheFactorization = true;
                    //blockSolvers[iPart].DefineMatrix(Block);
                    //blockSolvers[iPart] = new FullDirectSolver();
                    //blockSolvers[iPart].DefineMatrix(Block);

                    blockSolvers[iPart] = new ilPSP.LinSolvers.MUMPS.MUMPSSolver();
                    blockSolvers[iPart].DefineMatrix(Block);
                }
            }

            this.MtxFull = new ilPSP.LinSolvers.monkey.CPU.RefMatrix(M.ToMsrMatrix());

            if (CoarseSolver != null)
            {
                CoarseSolver.Init(op.CoarserLevel);
            }
        }
Example #11
0
        public override void SolverDriver <S>(CoordinateVector SolutionVec, S RHS)
        {
            using (var tr = new FuncTrace()) {
                m_SolutionVec = SolutionVec;

                int itc;
                itc = 0;
                double[] x, xt, xOld, f0, deltaX, ft;
                double   rat;
                double   alpha = 1E-4, sigma0 = 0.1, sigma1 = 0.5, maxarm = 20, gamma = 0.9;

                // Eval_F0
                using (new BlockTrace("Slv Init", tr)) {
                    base.Init(SolutionVec, RHS, out x, out f0);
                };

                Console.WriteLine("Residual base.init:   " + f0.L2NormPow2().MPISum().Sqrt());


                deltaX = new double[x.Length];
                xt     = new double[x.Length];
                ft     = new double[x.Length];


                this.CurrentLin.TransformSolFrom(SolutionVec, x);
                base.EvalResidual(x, ref f0);


                // fnorm
                double    fnorm  = f0.L2NormPow2().MPISum().Sqrt();
                double    fNormo = 1;
                double    errstep;
                double[]  step    = new double[x.Length];
                double[]  stepOld = new double[x.Length];
                MsrMatrix CurrentJac;

                Console.WriteLine("Start residuum for nonlinear iteration:  " + fnorm);

                OnIterationCallback(itc, x.CloneAs(), f0.CloneAs(), this.CurrentLin);

                //int[] Velocity_idx = SolutionVec.Mapping.GetSubvectorIndices(false, 0, 1, 2);
                //int[] Stresses_idx = SolutionVec.Mapping.GetSubvectorIndices(false, 3, 4, 5);

                //int[] Velocity_fields = new int[] { 0, 1, 2 };
                //int[] Stress_fields = new int[] { 3, 4, 5 };

                //int NoCoupledIterations = 1;

                using (new BlockTrace("Slv Iter", tr)) {
                    while (fnorm > ConvCrit && itc < MaxIter)
                    {
                        rat    = fnorm / fNormo;
                        fNormo = fnorm;
                        itc++;

                        // How should the inverse of the Jacobian be approximated?
                        if (ApproxJac == ApproxInvJacobianOptions.GMRES)
                        {
                            CurrentJac = new MsrMatrix(x.Length);
                            if (Precond != null)
                            {
                                Precond.Init(CurrentLin);
                            }
                            step = Krylov(SolutionVec, x, f0, out errstep);
                        }
                        else if (ApproxJac == ApproxInvJacobianOptions.DirectSolver)
                        {
                            CurrentJac = diffjac(SolutionVec, x, f0);
                            CurrentJac.SaveToTextFileSparse("Jacobi");
                            CurrentLin.OperatorMatrix.SaveToTextFileSparse("OpMatrix");
                            var solver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver();
                            solver.DefineMatrix(CurrentJac);
                            step.ClearEntries();
                            solver.Solve(step, f0);
                        }
                        else if (ApproxJac == ApproxInvJacobianOptions.DirectSolverHybrid)
                        {
                            //EXPERIMENTAL_____________________________________________________________________
                            MultidimensionalArray OpMatrixMatl = MultidimensionalArray.Create(x.Length, x.Length);
                            CurrentJac = diffjac(SolutionVec, x, f0);
                            //Console.WriteLine("Calling MATLAB/Octave...");
                            using (BatchmodeConnector bmc = new BatchmodeConnector())
                            {
                                bmc.PutSparseMatrix(CurrentJac, "Jacobi");
                                bmc.PutSparseMatrix(CurrentLin.OperatorMatrix, "OpMatrix");
                                bmc.Cmd("Jacobi(abs(Jacobi) < 10^-6)=0; dim = length(OpMatrix);");
                                bmc.Cmd("dim = length(OpMatrix);");
                                bmc.Cmd(@"for i=1:dim

                                if (i >= 16) && (i <= 33) && (i + 6 <= 33) && (i + 12 <= 33)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end

                                if (i >= 49) && (i <= 66) && (i + 6 <= 66) && (i + 12 <= 66)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end

                                if (i >= 82) && (i <= 99) && (i + 6 <= 99) && (i + 12 <= 99)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end

                                if (i >= 115) && (i <= 132) && (i + 6 <= 132) && (i + 12 <= 132)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end

                                if (i >= 148) && (i <= 165) && (i + 6 <= 165) && (i + 12 <= 165)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end 

                                if (i >= 181) && (i <= 198) && (i + 6 <= 198) && (i + 12 <= 198)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end  

                                if (i >= 214) && (i <= 231) && (i + 6 <= 231) && (i + 12 <= 231)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end
                                if (i >= 247) && (i <= 264) && (i + 6 <= 264) && (i + 12 <= 264)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end
                                if (i >= 280) && (i <= 297) && (i + 6 <= 297) && (i + 12 <= 297)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end
                                if (i >= 313) && (i <= 330) && (i + 6 <= 330) && (i + 12 <= 330)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end
                                if (i >= 346) && (i <= 363) && (i + 6 <= 363) && (i + 12 <= 363)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end
                                if (i >= 379) && (i <= 396) && (i + 6 <= 396) && (i + 12 <= 396)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end
                                if (i >= 412) && (i <= 429) && (i + 6 <= 429) && (i + 12 <= 429)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end
                                if (i >= 445) && (i <= 462) && (i + 6 <= 462) && (i + 12 <= 462)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end
                                if (i >= 478) && (i <= 495) && (i + 6 <= 495) && (i + 12 <= 495)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end
                                if (i >= 511) && (i <= 528) && (i + 6 <= 528) && (i + 12 <= 528)
                                    OpMatrix(i, i) = Jacobi(i, i);
                                    OpMatrix(i, i + 6) = Jacobi(i, i + 6);
                                    OpMatrix(i + 6, i) = Jacobi(i + 6, i);
                                    OpMatrix(i + 12, i + 6) = Jacobi(i + 12, i + 6);
                                    OpMatrix(i + 6, i + 12) = Jacobi(i + 6, i + 12);
                                end
                                end");
                                bmc.Cmd("OpMatrix = full(OpMatrix)");
                                bmc.GetMatrix(OpMatrixMatl, "OpMatrix");
                                bmc.Execute(false);
                            }

                            MsrMatrix OpMatrix = OpMatrixMatl.ToMsrMatrix();
                            var       solver   = new ilPSP.LinSolvers.MUMPS.MUMPSSolver();

                            //Console.WriteLine("USING HIGH EXPERIMENTAL OPMATRIX WITH JAC! only for p=1, GridLevel=2");
                            solver.DefineMatrix(OpMatrix);
                            //______________________________________________________________________________________________________

                            step.ClearEntries();
                            solver.Solve(step, f0);
                        }
                        else if (ApproxJac == ApproxInvJacobianOptions.DirectSolverOpMatrix)
                        {
                            CurrentJac = new MsrMatrix(x.Length);
                            CurrentJac = CurrentLin.OperatorMatrix.ToMsrMatrix();
                            var solver = new ilPSP.LinSolvers.MUMPS.MUMPSSolver();
                            solver.DefineMatrix(CurrentJac);
                            step.ClearEntries();
                            solver.Solve(step, f0);
                        }
                        else
                        {
                            throw new NotImplementedException("Your approximation option for the jacobian seems not to be existent.");
                        }

                        //if (itc > NoCoupledIterations)
                        //{
                        //    if (solveVelocity)
                        //    {
                        //        Console.WriteLine("stress correction = 0");
                        //        foreach (int idx in Stresses_idx)
                        //        {
                        //            step[idx] = 0.0;
                        //        }
                        //    }
                        //    else
                        //    {
                        //        Console.WriteLine("velocity correction = 0");
                        //        foreach (int idx in Velocity_idx)
                        //        {
                        //            step[idx] = 0.0;
                        //        }
                        //    }
                        //}

                        // Start line search
                        xOld = x;
                        double lambda = 1;
                        double lamm   = 1;
                        double lamc   = lambda;
                        double iarm   = 0;
                        xt = x.CloneAs();
                        xt.AccV(lambda, step);
                        this.CurrentLin.TransformSolFrom(SolutionVec, xt);
                        EvaluateOperator(1, SolutionVec.Mapping.Fields, ft);
                        var nft = ft.L2NormPow2().MPISum().Sqrt(); var nf0 = f0.L2NormPow2().MPISum().Sqrt(); var ff0 = nf0 * nf0; var ffc = nft * nft; var ffm = nft * nft;

                        // Control of the the step size
                        while (nft >= (1 - alpha * lambda) * nf0 && iarm < maxStep)
                        {
                            // Line search starts here

                            if (iarm == 0)
                            {
                                lambda = sigma1 * lambda;
                            }
                            else
                            {
                                lambda = parab3p(lamc, lamm, ff0, ffc, ffm);
                            }

                            // Update x;
                            xt = x.CloneAs();
                            xt.AccV(lambda, step);
                            lamm = lamc;
                            lamc = lambda;

                            this.CurrentLin.TransformSolFrom(SolutionVec, xt);

                            EvaluateOperator(1, SolutionVec.Mapping.Fields, ft);

                            nft = ft.L2NormPow2().MPISum().Sqrt();
                            ffm = ffc;
                            ffc = nft * nft;
                            iarm++;

#if DEBUG
                            Console.WriteLine("Step size:  " + lambda + "with Residuum:  " + nft);
#endif
                        }
                        // transform solution back to 'original domain'
                        // to perform the linearization at the new point...
                        // (and for Level-Set-Updates ...)
                        this.CurrentLin.TransformSolFrom(SolutionVec, xt);

                        // update linearization
                        base.Update(SolutionVec.Mapping.Fields, ref xt);

                        // residual evaluation & callback
                        base.EvalResidual(xt, ref ft);

                        // EvaluateOperator(1, SolutionVec.Mapping.Fields, ft);

                        //base.Init(SolutionVec, RHS, out x, out f0);

                        fnorm = ft.L2NormPow2().MPISum().Sqrt();

                        x  = xt;
                        f0 = ft.CloneAs();

                        //if (itc > NoCoupledIterations)
                        //{

                        //    double coupledL2Res = 0.0;
                        //    if (solveVelocity)
                        //    {
                        //        foreach (int idx in Velocity_idx)
                        //        {
                        //            coupledL2Res += f0[idx].Pow2();
                        //        }
                        //    }
                        //    else
                        //    {
                        //        foreach (int idx in Stresses_idx)
                        //        {
                        //            coupledL2Res += f0[idx].Pow2();
                        //        }
                        //    }
                        //    coupledL2Res = coupledL2Res.Sqrt();

                        //    Console.WriteLine("coupled residual = {0}", coupledL2Res);

                        //    if (solveVelocity && coupledL2Res < this.VelocitySolver_ConvergenceCriterion)
                        //    {
                        //        Console.WriteLine("SolveVelocity = false");
                        //        this.solveVelocity = false;
                        //    }
                        //    else if (!solveVelocity && coupledL2Res < this.StressSolver_ConvergenceCriterion)
                        //    {
                        //        Console.WriteLine("SolveVelocity = true");
                        //        this.solveVelocity = true;
                        //    }
                        //}

                        OnIterationCallback(itc, x.CloneAs(), f0.CloneAs(), this.CurrentLin);
                    }
                }

                SolutionVec = m_SolutionVec;
            }
        }