/// <summary> /// defines how species in the agglomerated cells /// map to species in the base grid cells. /// </summary> private void UpdateSpeciesMapping(MultiphaseCellAgglomerator Agglomerator) { Debug.Assert(object.ReferenceEquals(Agglomerator.Tracker, this.XDGBasis.Tracker)); var LsTrk = this.XDGBasis.Tracker; int GlobalNoOfSpc = LsTrk.SpeciesIdS.Count; var agg = base.AggGrid; int[][] compCells = agg.iLogicalCells.AggregateCellToParts; int JAGG = compCells.Length; Debug.Assert(JAGG == agg.iLogicalCells.Count); int JAGGup = agg.iLogicalCells.NoOfLocalUpdatedCells; this.UsedSpecies = Agglomerator.SpeciesList.ToArray(); if (NoOfSpecies == null) { NoOfSpecies = new int[JAGG]; SpeciesIndexMapping = new int[JAGG][, ]; } else { Debug.Assert(NoOfSpecies.Length == JAGG); Array.Clear(NoOfSpecies, 0, JAGG); } if (AggCellsSpecies == null) { AggCellsSpecies = new SpeciesId[JAGG][]; } //if (CoarseToFineSpeciesIndex == null && agg.MgLevel > 0) // CoarseToFineSpeciesIndex = new int[JAGG][,]; Dictionary <SpeciesId, BitArray> agglomeratedCells = new Dictionary <SpeciesId, BitArray>(); foreach (var SpId in Agglomerator.SpeciesList) { agglomeratedCells.Add(SpId, Agglomerator.GetAgglomerator(SpId).AggInfo.SourceCells.GetBitMaskWithExternal()); } // temp buffer int[] _NoOfSpecies = new int[agg.jCellCoarse2jCellFine[0].Length]; ReducedRegionCode[] _RRcs = new ReducedRegionCode[agg.iLogicalCells.AggregateCellToParts[0].Length]; SpeciesId[] allPresentSpecies = new SpeciesId[LsTrk.TotalNoOfSpecies]; // loop over all aggregate (multigrid) cells... for (int jagg = 0; jagg < JAGGup; jagg++) { int[] compCell = compCells[jagg]; int K = compCell.Length; // buffer sizes if (K > _NoOfSpecies.Length) { int newLen = (int)Math.Ceiling(K * 1.2); Array.Resize(ref _NoOfSpecies, newLen); Array.Resize(ref _RRcs, newLen); } // check no of species: int MaxNoOfSpecies = 0; // number of species in the composite cell for (int k = 0; k < K; k++) // loop over original cells in composite cell { int jCell = compCell[k]; _NoOfSpecies[k] = LsTrk.Regions.GetNoOfSpecies(jCell, out _RRcs[k]); MaxNoOfSpecies = Math.Max(_NoOfSpecies[k], MaxNoOfSpecies); } if (MaxNoOfSpecies == 1) { // only one species -- use single-phase implementation in underlying class this.SpeciesIndexMapping[jagg] = null; this.NoOfSpecies[jagg] = 1; //this.XCompositeBasis[jagg] = null; this.AggCellsSpecies[jagg] = null; //this.CoarseToFineSpeciesIndex[jagg] = null; } else { //LsTrk.ContainesSpecies int w = 0; // counter for found species if (this.SpeciesIndexMapping[jagg] == null || this.SpeciesIndexMapping[jagg].GetLength(0) != w) { this.SpeciesIndexMapping[jagg] = new int[0, compCell.Length]; } ArrayTools.SetAll(this.SpeciesIndexMapping[jagg], -897645); // collect all species which are present in the composite cell 'jagg': // ------------------------------------------------------------------- for (int k = 0; k < K; k++) // loop over all base cells in the aggregate cell 'jagg' { var rrc = _RRcs[k]; int jCell = compCell[k]; // base cell index for (int iSpc = _NoOfSpecies[k] - 1; iSpc >= 0; iSpc--) { SpeciesId spId = LsTrk.GetSpeciesIdFromIndex(rrc, iSpc); bool isPresent = LsTrk.Regions.IsSpeciesPresentInCell(spId, jCell); bool isAgglomerated = agglomeratedCells.ContainsKey(spId) ? agglomeratedCells[spId][jCell] : false; bool isUsed = Array.IndexOf(this.UsedSpecies, spId) >= 0; if (isUsed && isPresent && !isAgglomerated) { bool bfound = false; int z; for (z = 0; z < w; z++) // loop over species found so far... { if (allPresentSpecies[z] == spId) { bfound = true; break; } } if (!bfound) { // species not found -> add to list allPresentSpecies[w] = spId; z = w; w++; } // z is the species index in the composite cell // iSpc is the species index in 'jCell' Resize2DArray(ref this.SpeciesIndexMapping[jagg], w, K, -897645); this.SpeciesIndexMapping[jagg][z, k] = iSpc; } } } this.AggCellsSpecies[jagg] = allPresentSpecies.GetSubVector(0, w); this.NoOfSpecies[jagg] = w; Debug.Assert(this.NoOfSpecies[jagg] == this.SpeciesIndexMapping[jagg].GetLength(0)); } Debug.Assert(this.NoOfSpecies[jagg] >= 0); Debug.Assert(this.NoOfSpecies[jagg] <= this.UsedSpecies.Length); } // MPI exchange for external cells // note: external aggregation cells may be incomplete, i.e. not have all parts, only those known to parent NoOfSpecies.MPIExchange(this.AggGrid); }
protected override double RunSolverOneStep(int TimestepNo, double phystime, double dt) { Console.WriteLine(" Timestep # " + TimestepNo + ", phystime = " + phystime); //phystime = 1.8; LsUpdate(phystime); // operator-matrix assemblieren MsrMatrix OperatorMatrix = new MsrMatrix(u.Mapping, u.Mapping); double[] Affine = new double[OperatorMatrix.RowPartitioning.LocalLength]; // Agglomerator setup MultiphaseCellAgglomerator Agg = LsTrk.GetAgglomerator(new SpeciesId[] { LsTrk.GetSpeciesId("B") }, QuadOrder, this.THRESHOLD); // plausibility of cell length scales if (SER_PAR_COMPARISON) { TestLengthScales(QuadOrder, TimestepNo); } Console.WriteLine("Inter-Process agglomeration? " + Agg.GetAgglomerator(LsTrk.GetSpeciesId("B")).AggInfo.InterProcessAgglomeration); if (this.THRESHOLD > 0.01) { TestAgglomeration_Extraploation(Agg); TestAgglomeration_Projection(QuadOrder, Agg); } CheckExchange(true); CheckExchange(false); // operator matrix assembly XSpatialOperatorMk2.XEvaluatorLinear mtxBuilder = Op.GetMatrixBuilder(base.LsTrk, u.Mapping, null, u.Mapping); mtxBuilder.time = 0.0; mtxBuilder.ComputeMatrix(OperatorMatrix, Affine); Agg.ManipulateMatrixAndRHS(OperatorMatrix, Affine, u.Mapping, u.Mapping); // mass matrix factory var Mfact = LsTrk.GetXDGSpaceMetrics(new SpeciesId[] { LsTrk.GetSpeciesId("B") }, QuadOrder, 1).MassMatrixFactory;// new MassMatrixFactory(u.Basis, Agg); // Mass matrix/Inverse Mass matrix //var MassInv = Mfact.GetMassMatrix(u.Mapping, new double[] { 1.0 }, true, LsTrk.GetSpeciesId("B")); var Mass = Mfact.GetMassMatrix(u.Mapping, new double[] { 1.0 }, false, LsTrk.GetSpeciesId("B")); Agg.ManipulateMatrixAndRHS(Mass, default(double[]), u.Mapping, u.Mapping); var MassInv = Mass.InvertBlocks(OnlyDiagonal: true, Subblocks: true, ignoreEmptyBlocks: true, SymmetricalInversion: false); // test that operator depends only on B-species values double DepTest = LsTrk.Regions.GetSpeciesSubGrid("B").TestMatrixDependency(OperatorMatrix, u.Mapping, u.Mapping); Console.WriteLine("Matrix dependency test: " + DepTest); Assert.LessOrEqual(DepTest, 0.0); // diagnostic output Console.WriteLine("Number of Agglomerations (all species): " + Agg.TotalNumberOfAgglomerations); Console.WriteLine("Number of Agglomerations (species 'B'): " + Agg.GetAgglomerator(LsTrk.GetSpeciesId("B")).AggInfo.SourceCells.NoOfItemsLocally.MPISum()); // operator auswerten: double[] x = new double[Affine.Length]; BLAS.daxpy(x.Length, 1.0, Affine, 1, x, 1); OperatorMatrix.SpMVpara(1.0, u.CoordinateVector, 1.0, x); MassInv.SpMV(1.0, x, 0.0, du_dx.CoordinateVector); Agg.GetAgglomerator(LsTrk.GetSpeciesId("B")).Extrapolate(du_dx.Mapping); // markieren, wo ueberhaupt A und B sind Bmarker.AccConstant(1.0, LsTrk.Regions.GetSpeciesSubGrid("B").VolumeMask); Amarker.AccConstant(+1.0, LsTrk.Regions.GetSpeciesSubGrid("A").VolumeMask); if (usePhi0 && usePhi1) { Xmarker.AccConstant(+1.0, LsTrk.Regions.GetSpeciesSubGrid("X").VolumeMask); } // compute error ERR.Clear(); ERR.Acc(1.0, du_dx_Exact, LsTrk.Regions.GetSpeciesSubGrid("B").VolumeMask); ERR.Acc(-1.0, du_dx, LsTrk.Regions.GetSpeciesSubGrid("B").VolumeMask); double L2Err = ERR.L2Norm(LsTrk.Regions.GetSpeciesSubGrid("B").VolumeMask); Console.WriteLine("L2 Error: " + L2Err); XERR.Clear(); XERR.GetSpeciesShadowField("B").Acc(1.0, ERR, LsTrk.Regions.GetSpeciesSubGrid("B").VolumeMask); double xL2Err = XERR.L2Norm(); Console.WriteLine("L2 Error (in XDG space): " + xL2Err); // check error double ErrorThreshold = 1.0e-1; if (this.MomentFittingVariant == XQuadFactoryHelper.MomentFittingVariants.OneStepGaussAndStokes) { ErrorThreshold = 1.0e-6; // HMF is designed for such integrands and should perform close to machine accuracy; on general integrands, the precision is different. } bool IsPassed = ((L2Err <= ErrorThreshold || this.THRESHOLD <= ErrorThreshold) && xL2Err <= ErrorThreshold); if (IsPassed) { Console.WriteLine("Test PASSED"); } else { Console.WriteLine("Test FAILED: check errors."); //PlotCurrentState(phystime, TimestepNo, 3); } if (TimestepNo > 1) { if (this.THRESHOLD > ErrorThreshold) { // without agglomeration, the error in very tiny cut-cells may be large over the whole cell // However, the error in the XDG-space should be small under all circumstances Assert.LessOrEqual(L2Err, ErrorThreshold, "DG L2 error of computing du_dx"); } Assert.LessOrEqual(xL2Err, ErrorThreshold, "XDG L2 error of computing du_dx"); } // return/Ende base.NoOfTimesteps = 17; //base.NoOfTimesteps = 2; dt = 0.3; return(dt); }
private void AssembleMatrix(double MU_A, double MU_B, out BlockMsrMatrix M, out double[] b, out MultiphaseCellAgglomerator agg, out MassMatrixFactory massFact) { using (var tr = new FuncTrace()) { // create operator // =============== if (this.Control.SetDefaultDiriBndCnd) { this.Control.xLaplaceBCs.g_Diri = ((CommonParamsBnd inp) => 0.0); this.Control.xLaplaceBCs.IsDirichlet = (inp => true); } double D = this.GridData.SpatialDimension; int p = u.Basis.Degree; double penalty_base = (p + 1) * (p + D) / D; double penalty_multiplyer = base.Control.penalty_multiplyer; XQuadFactoryHelper.MomentFittingVariants momentFittingVariant; if (D == 3) { momentFittingVariant = XQuadFactoryHelper.MomentFittingVariants.Classic; } momentFittingVariant = this.Control.CutCellQuadratureType; int order = this.u.Basis.Degree * 2; XSpatialOperator Op = new XSpatialOperator(1, 1, (A, B, C) => order, "u", "c1"); var lengthScales = ((BoSSS.Foundation.Grid.Classic.GridData)GridData).Cells.PenaltyLengthScales; var lap = new XLaplace_Bulk(this.LsTrk, penalty_multiplyer * penalty_base, "u", this.Control.xLaplaceBCs, 1.0, MU_A, MU_B, lengthScales, this.Control.ViscosityMode); Op.EquationComponents["c1"].Add(lap); // Bulk form Op.EquationComponents["c1"].Add(new XLaplace_Interface(this.LsTrk, MU_A, MU_B, penalty_base * 2, this.Control.ViscosityMode)); // coupling form Op.Commit(); // create agglomeration // ==================== var map = new UnsetteledCoordinateMapping(u.Basis); //agg = new MultiphaseCellAgglomerator( // new CutCellMetrics(momentFittingVariant, // QuadOrderFunc.SumOfMaxDegrees(RoundUp: true)(map.BasisS.Select(bs => bs.Degree).ToArray(), new int[0], map.BasisS.Select(bs => bs.Degree).ToArray()), // //this.HMFDegree, // LsTrk, this.LsTrk.SpeciesIdS.ToArray()), // this.Control.AgglomerationThreshold, false); agg = LsTrk.GetAgglomerator(this.LsTrk.SpeciesIdS.ToArray(), order, this.Control.AgglomerationThreshold); // compute matrix // ============= using (new BlockTrace("XdgMatrixAssembly", tr)) { M = new BlockMsrMatrix(map, map); b = new double[M.RowPartitioning.LocalLength]; Op.ComputeMatrixEx(LsTrk, map, null, map, M, b, false, 0.0, true, agg.CellLengthScales, null, null, //out massFact, this.LsTrk.SpeciesIdS.ToArray()); } // compare with linear evaluation // ============================== DGField[] testDomainFieldS = map.BasisS.Select(bb => new XDGField(bb as XDGBasis)).ToArray(); CoordinateVector test = new CoordinateVector(testDomainFieldS); DGField[] errFieldS = map.BasisS.Select(bb => new XDGField(bb as XDGBasis)).ToArray(); CoordinateVector Err = new CoordinateVector(errFieldS); var eval = Op.GetEvaluatorEx(LsTrk, testDomainFieldS, null, map); foreach (var s in this.LsTrk.SpeciesIdS) { eval.SpeciesOperatorCoefficients[s].CellLengthScales = agg.CellLengthScales[s]; } eval.time = 0.0; int L = test.Count; Random r = new Random(); for (int i = 0; i < L; i++) { test[i] = r.NextDouble(); } double[] R1 = new double[L]; double[] R2 = new double[L]; eval.Evaluate(1.0, 1.0, R1); R2.AccV(1.0, b); M.SpMV(1.0, test, 1.0, R2); Err.AccV(+1.0, R1); Err.AccV(-1.0, R2); double ErrDist = GenericBlas.L2DistPow2(R1, R2).MPISum().Sqrt(); double Ref = test.L2NormPow2().MPISum().Sqrt(); Assert.LessOrEqual(ErrDist, Ref * 1.0e-5, "Mismatch between explicit evaluation of XDG operator and matrix."); // agglomeration wahnsinn // ====================== agg.ManipulateMatrixAndRHS(M, b, map, map); foreach (var S in this.LsTrk.SpeciesNames) { Console.WriteLine(" Species {0}: no of agglomerated cells: {1}", S, agg.GetAgglomerator(this.LsTrk.GetSpeciesId(S)).AggInfo.SourceCells.NoOfItemsLocally); } // mass matrix factory // =================== Basis maxB = map.BasisS.ElementAtMax(bss => bss.Degree); //massFact = new MassMatrixFactory(maxB, agg); massFact = LsTrk.GetXDGSpaceMetrics(this.LsTrk.SpeciesIdS.ToArray(), order).MassMatrixFactory; } }
/// <summary> /// Mainly, comparison of single-core vs MPI-parallel run /// </summary> void TestLengthScales(int quadOrder, int TimestepNo) { string name_disc = $"t{TimestepNo}-alpha{this.THRESHOLD}-p{this.DEGREE}-q{MomentFittingVariant}"; var spcA = LsTrk.GetSpeciesId("A"); var spcB = LsTrk.GetSpeciesId("B"); var species = new[] { spcA, spcB }; //MultiphaseCellAgglomerator.CheckFile = $"InsideMpagg-{name_disc}.csv"; MultiphaseCellAgglomerator Agg = LsTrk.GetAgglomerator(species, quadOrder, this.THRESHOLD); int RefMPIsize = 1; // check level-set coordinates // =========================== { var LsChecker = new TestingIO(this.GridData, $"LevelSets-{name_disc}.csv", RefMPIsize); LsChecker.AddDGField(this.Phi0); LsChecker.AddDGField(this.Phi1); LsChecker.DoIOnow(); Assert.Less(LsChecker.AbsError(this.Phi0), 1.0e-8, "Mismatch in level-set 0 between single-core and parallel run."); Assert.Less(LsChecker.AbsError(this.Phi1), 1.0e-8, "Mismatch in level-set 1 between single-core and parallel run."); } // check equality of agglomeration // =============================== // Note: agglomeration in parallel and serial mode is not necessarily equal - but very often it is. // Therefore, we need to check whether the agglomeration is equal or not, // in order to know whether agglomerated length scales should be compared or not. bool[] equalAggAsinReferenceRun; { var aggoCheck = new TestingIO(this.GridData, $"Agglom-{name_disc}.csv", RefMPIsize); long[] GiDs = GridData.CurrentGlobalIdPermutation.Values; int J = GridData.iLogicalCells.NoOfLocalUpdatedCells; long[] extGiDs = GridData.iParallel.GlobalIndicesExternalCells; for (int iSpc = 0; iSpc < species.Length; iSpc++) { var spc = species[iSpc]; var spcN = LsTrk.GetSpeciesName(spc); var ai = Agg.GetAgglomerator(spc).AggInfo; var srcMask = ai.SourceCells.GetBitMask(); var aggPairs = ai.AgglomerationPairs; aggoCheck.AddColumn($"SourceCells{spcN}", delegate(double[] X, int j, int jG) { if (srcMask[j]) { var pair = aggPairs.Single(cap => cap.jCellSource == j); if (pair.jCellTarget < J) { return((double)GiDs[pair.jCellTarget]); } else { return((double)extGiDs[pair.jCellTarget - J]); } } return(0.0); }); } aggoCheck.DoIOnow(); equalAggAsinReferenceRun = new bool[species.Length]; for (int iSpc = 0; iSpc < species.Length; iSpc++) { var spc = species[iSpc]; var spcN = LsTrk.GetSpeciesName(spc); equalAggAsinReferenceRun[iSpc] = aggoCheck.AbsError($"SourceCells{spcN}") < 0.1; if (equalAggAsinReferenceRun[iSpc] == false) { Console.WriteLine("Different agglomeration between single-core and parallel run for species " + spcN + "."); } } //Agg.PlotAgglomerationPairs($"-aggPairs-{name_disc}-MPI{MPIRank + 1}of{MPISize}"); } // compare length scales // ===================== csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out int rank); foreach (ICutCellMetrics ccm in new ICutCellMetrics[] { Agg.NonAgglomeratedMetrics, Agg }) // loop over non-agglom and agglomerated metrics { string name; bool Agglom; if (object.ReferenceEquals(ccm, Agg)) { name = "Agglomerated"; Agglom = true; } else { name = "Nonagglom"; Agglom = false; } MultidimensionalArray CellSurfaceA = ccm.CellSurface[spcA]; MultidimensionalArray CellVolumeA = ccm.CutCellVolumes[spcA]; MultidimensionalArray CellSurfaceB = ccm.CellSurface[spcB]; MultidimensionalArray CellVolumeB = ccm.CutCellVolumes[spcB]; string FileName = $"{name}LengthScales-{name_disc}.csv"; var Checker = new TestingIO(this.GridData, FileName, RefMPIsize); Checker.AddColumn("CellSurfA", (double[] X, int j, int jG) => CellSurfaceA[j]); Checker.AddColumn("CellVolA", (double[] X, int j, int jG) => CellVolumeA[j]); Checker.AddColumn("CellSurfB", (double[] X, int j, int jG) => CellSurfaceB[j]); Checker.AddColumn("CellVolB", (double[] X, int j, int jG) => CellVolumeB[j]); Checker.DoIOnow(); if (this.MPISize == 1) { var Checker2 = new TestingIO(this.GridData, FileName, int.MaxValue); Checker2.AddColumn("CellSurfA", (double[] X, int j, int jG) => CellSurfaceA[j]); Checker2.AddColumn("CellVolA", (double[] X, int j, int jG) => CellVolumeA[j]); Checker2.AddColumn("CellSurfB", (double[] X, int j, int jG) => CellSurfaceB[j]); Checker2.AddColumn("CellVolB", (double[] X, int j, int jG) => CellVolumeB[j]); Checker2.DoIOnow(); foreach (string s in Checker2.ColumnNames) { Assert.Less(Checker2.RelError(s), 1.0e-10, "'TestingUtils.Compare' itself is f****d up."); } } double srfA = Checker.RelError("CellSurfA") * ((!Agglom || equalAggAsinReferenceRun[0]) ? 1.0 : 0.0); double volA = Checker.RelError("CellVolA") * ((!Agglom || equalAggAsinReferenceRun[0]) ? 1.0 : 0.0); double srfB = Checker.RelError("CellSurfB") * ((!Agglom || equalAggAsinReferenceRun[1]) ? 1.0 : 0.0); double volB = Checker.RelError("CellVolB") * ((!Agglom || equalAggAsinReferenceRun[1]) ? 1.0 : 0.0); if (srfA + volA + srfB + volB > 0.001) { Console.WriteLine($"Mismatch in {name} cell surface for species A between single-core and parallel run: {srfA}."); Console.WriteLine($"Mismatch in {name} cell volume for species A between single-core and parallel run: {volA}."); Console.WriteLine($"Mismatch in {name} cell surface for species B between single-core and parallel run: {srfB}."); Console.WriteLine($"Mismatch in {name} cell volume for species B between single-core and parallel run: {volB}."); } Assert.Less(srfA, BLAS.MachineEps.Sqrt(), $"Mismatch in {name} cell surface for species A between single-core and parallel run."); Assert.Less(volA, BLAS.MachineEps.Sqrt(), $"Mismatch in {name} cell volume for species A between single-core and parallel run."); Assert.Less(srfB, BLAS.MachineEps.Sqrt(), $"Mismatch in {name} cell surface for species B between single-core and parallel run."); Assert.Less(volB, BLAS.MachineEps.Sqrt(), $"Mismatch in {name} cell volume for species B between single-core and parallel run."); } }