/// <summary>
        /// computes the inner product of this array and another array <paramref name="other"/>.
        /// </summary>
        /// <param name="other">
        /// must be of the same size a this array
        /// </param>
        /// <returns></returns>
        public double InnerProduct(MultidimensionalArray other)
        {
            if (this.Dimension != other.Dimension)
            {
                throw new ArgumentException("mismatch in number of dimensions");
            }
            int D = this.Dimension;

            for (int k = 0; k < D; k++)
            {
                if (this.GetLength(k) != other.GetLength(k))
                {
                    throw new ArgumentException("mismatch in dimension " + k);
                }
            }

            if (this.IsContinious && other.IsContinious)
            {
                unsafe
                {
                    fixed(double *pStorageThis = this.m_Storage, pStorageOther = other.m_Storage)
                    {
                        return(BLAS.ddot(this.Length, pStorageThis, 1, pStorageOther, 1));
                    }
                }
            }
            else
            {
                double acc = 0;
                this.ApplyAll(delegate(int[] index, ref double entry) {
                    acc += entry * other[index];
                });
                return(acc);
            }
        }
        void __AddSol(ref double[] X)
        {
            Debug.Assert(SolHistory.Count == MxxHistory.Count);
            Debug.Assert(X.Length == OpMatrix._RowPartitioning.LocalLength);
            int L         = X.Length;
            int KrylovDim = SolHistory.Count;

            double[] Mxx = new double[L];
            OpMatrix.SpMV(1.0, X, 0.0, Mxx);

            for (int i = 0; i < KrylovDim; i++)
            {
                Debug.Assert(!object.ReferenceEquals(Mxx, MxxHistory[i]));
                double beta = BLAS.ddot(L, Mxx, 1, MxxHistory[i], 1).MPISum();
                BLAS.daxpy(L, -beta, SolHistory[i], 1, X, 1);
                BLAS.daxpy(L, -beta, MxxHistory[i], 1, Mxx, 1);
            }

            double gamma = 1.0 / Mxx.L2NormPow2().MPISum().Sqrt();

            //double gamma = 1.0 / BLAS.dnrm2(L, Mxx, 1).Pow2().MPISum().Sqrt();
            BLAS.dscal(L, gamma, Mxx, 1);
            BLAS.dscal(L, gamma, X, 1);

            SolHistory.Add(X);
            MxxHistory.Add(Mxx);
            X = null;
        }
        double MinimizeResidual(double[] outX, double[] Sol0, double[] Res0, double[] outRes, bool diagnosis = false)
        {
            using (new FuncTrace()) {
                Debug.Assert(SolHistory.Count == MxxHistory.Count);
                Debug.Assert(outX.Length == m_MgOperator.Mapping.LocalLength);
                Debug.Assert(Sol0.Length == m_MgOperator.Mapping.LocalLength);
                Debug.Assert(Res0.Length == m_MgOperator.Mapping.LocalLength);
                Debug.Assert(outRes.Length == m_MgOperator.Mapping.LocalLength);

                int KrylovDim = SolHistory.Count;
                int L         = outX.Length;

                double[] alpha = new double[KrylovDim];
                for (int i = 0; i < KrylovDim; i++)
                {
                    //alpha[i] = GenericBlas.InnerProd(MxxHistory[i], Res0).MPISum();
                    alpha[i] = BLAS.ddot(L, MxxHistory[i], 1, Res0, 1);
                }
                alpha = alpha.MPISum();


                Array.Copy(Sol0, outX, L);
                Array.Copy(Res0, outRes, L);
                for (int i = 0; i < KrylovDim; i++)
                {
                    //outX.AccV(alpha[i], SolHistory[i]);
                    //outRes.AccV(-alpha[i], MxxHistory[i]);
                    BLAS.daxpy(L, alpha[i], SolHistory[i], 1, outX, 1);
                    BLAS.daxpy(L, -alpha[i], MxxHistory[i], 1, outRes, 1);
                }

                double ResNorm = BLAS.dnrm2(L, outRes, 1).Pow2().MPISum().Sqrt();

                //Console.WriteLine("OrthonormalizationMultigrid: minimizing ofer " + KrylovDim + " vectors");


                return(ResNorm);

                /* we cannot do the following
                 * // since the 'MxxHistory' vectors form an orthonormal system,
                 * // the L2-norm is the L2-Norm of the 'alpha'-coordinates (Parceval's equality)
                 * return alpha.L2Norm();
                 */
            }
        }
        void AddSolCore(ref double[] X)
        {
            Debug.Assert(SolHistory.Count == MxxHistory.Count);
            Debug.Assert(X.Length == OpMatrix._RowPartitioning.LocalLength);
            int L         = X.Length;
            int KrylovDim = SolHistory.Count;

            double[] Mxx = new double[L];
            OpMatrix.SpMV(1.0, X, 0.0, Mxx);

            //double NormInitial = Mxx.MPI_L2Norm();

            for (int jj = 0; jj < 2; jj++)   // re-orthogonalisation, loop-limit to 2; See book of Saad, p 162, section 6.3.2
            {
                for (int i = 0; i < KrylovDim; i++)
                {
                    Debug.Assert(!object.ReferenceEquals(Mxx, MxxHistory[i]));
                    double beta = BLAS.ddot(L, Mxx, 1, MxxHistory[i], 1).MPISum();
                    BLAS.daxpy(L, -beta, SolHistory[i], 1, X, 1);
                    BLAS.daxpy(L, -beta, MxxHistory[i], 1, Mxx, 1);
                }

                //double NormAfter = Mxx.MPI_L2Norm();
                //Console.WriteLine("   orthonormalization norm reduction: " + (NormAfter/NormInitial));

                double gamma = 1.0 / Mxx.L2NormPow2().MPISum().Sqrt();
                //double gamma = 1.0 / BLAS.dnrm2(L, Mxx, 1).Pow2().MPISum().Sqrt();
                BLAS.dscal(L, gamma, Mxx, 1);
                BLAS.dscal(L, gamma, X, 1);
            }


            SolHistory.Add(X);
            MxxHistory.Add(Mxx);
            X = null;
        }
        double MinimizeResidual(double[] outX, double[] Sol0, double[] Res0, double[] outRes, bool diagnosis = false)
        {
            using (new FuncTrace()) {
                Debug.Assert(SolHistory.Count == MxxHistory.Count);
                Debug.Assert(outX.Length == m_MgOperator.Mapping.LocalLength);
                Debug.Assert(Sol0.Length == m_MgOperator.Mapping.LocalLength);
                Debug.Assert(Res0.Length == m_MgOperator.Mapping.LocalLength);
                Debug.Assert(outRes.Length == m_MgOperator.Mapping.LocalLength);

                int KrylovDim = SolHistory.Count;
                int L         = outX.Length;

                double[] alpha = new double[KrylovDim];
                for (int i = 0; i < KrylovDim; i++)
                {
                    //alpha[i] = GenericBlas.InnerProd(MxxHistory[i], Res0).MPISum();
                    alpha[i] = BLAS.ddot(L, MxxHistory[i], 1, Res0, 1);
                }
                alpha = alpha.MPISum();



                //outX.SetV(Sol0);
                //outRes.SetV(Res0);
                Array.Copy(Sol0, outX, L);
                Array.Copy(Res0, outRes, L);
                for (int i = 0; i < KrylovDim; i++)
                {
                    //outX.AccV(alpha[i], SolHistory[i]);
                    //outRes.AccV(-alpha[i], MxxHistory[i]);
                    BLAS.daxpy(L, alpha[i], SolHistory[i], 1, outX, 1);
                    BLAS.daxpy(L, -alpha[i], MxxHistory[i], 1, outRes, 1);
                }

                double ResNorm = BLAS.dnrm2(L, outRes, 1).Pow2().MPISum().Sqrt();


                /*
                 * if(ResNorm < Tolerance) {
                 *  alpha.SaveToStream(Console.Out, m_MgOperator.BaseGridProblemMapping.MPI_Comm);
                 *
                 *  alpha.SaveToTextFile("ConvScheisse.txt", m_MgOperator.BaseGridProblemMapping.MPI_Comm);
                 * }
                 */

                /*
                 * if (diagnosis) {
                 *  for (int i = 0; i < KrylovDim; i++) {
                 *      Console.WriteLine("      s " + alpha[KrylovDim - i - 1]);
                 *
                 *  }
                 * }
                 */

                return(ResNorm);

                /* we cannot do the following
                 * // since the 'MxxHistory' vectors form an orthonormal system,
                 * // the L2-norm is the L2-Norm of the 'alpha'-coordinates (Parceval's equality)
                 * return alpha.L2Norm();
                 */
            }
        }