/// <summary>
/// defines how species in the agglomerated cells
/// map to species in the base grid cells.
/// </summary>
private void UpdateSpeciesMapping(MultiphaseCellAgglomerator Agglomerator)
{
Debug.Assert(object.ReferenceEquals(Agglomerator.Tracker, this.XDGBasis.Tracker));
var LsTrk = this.XDGBasis.Tracker;
int GlobalNoOfSpc = LsTrk.SpeciesIdS.Count;
var agg = base.AggGrid;
int[][] compCells = agg.iLogicalCells.AggregateCellToParts;
int JAGG = compCells.Length;
Debug.Assert(JAGG == agg.iLogicalCells.Count);
int JAGGup = agg.iLogicalCells.NoOfLocalUpdatedCells;
this.UsedSpecies = Agglomerator.SpeciesList.ToArray();
if (NoOfSpecies == null)
{
NoOfSpecies = new int[JAGG];
SpeciesIndexMapping = new int[JAGG][, ];
}
else
{
Debug.Assert(NoOfSpecies.Length == JAGG);
Array.Clear(NoOfSpecies, 0, JAGG);
}
if (AggCellsSpecies == null)
{
AggCellsSpecies = new SpeciesId[JAGG][];
}
//if (CoarseToFineSpeciesIndex == null && agg.MgLevel > 0)
// CoarseToFineSpeciesIndex = new int[JAGG][,];
Dictionary <SpeciesId, BitArray> agglomeratedCells = new Dictionary <SpeciesId, BitArray>();
foreach (var SpId in Agglomerator.SpeciesList)
{
agglomeratedCells.Add(SpId, Agglomerator.GetAgglomerator(SpId).AggInfo.SourceCells.GetBitMaskWithExternal());
}
// temp buffer
int[] _NoOfSpecies = new int[agg.jCellCoarse2jCellFine[0].Length];
ReducedRegionCode[] _RRcs = new ReducedRegionCode[agg.iLogicalCells.AggregateCellToParts[0].Length];
SpeciesId[] allPresentSpecies = new SpeciesId[LsTrk.TotalNoOfSpecies];
// loop over all aggregate (multigrid) cells...
for (int jagg = 0; jagg < JAGGup; jagg++)
{
int[] compCell = compCells[jagg];
int K = compCell.Length;
// buffer sizes
if (K > _NoOfSpecies.Length)
{
int newLen = (int)Math.Ceiling(K * 1.2);
Array.Resize(ref _NoOfSpecies, newLen);
Array.Resize(ref _RRcs, newLen);
}
// check no of species:
int MaxNoOfSpecies = 0; // number of species in the composite cell
for (int k = 0; k < K; k++) // loop over original cells in composite cell
{
int jCell = compCell[k];
_NoOfSpecies[k] = LsTrk.Regions.GetNoOfSpecies(jCell, out _RRcs[k]);
MaxNoOfSpecies = Math.Max(_NoOfSpecies[k], MaxNoOfSpecies);
}
if (MaxNoOfSpecies == 1)
{
// only one species -- use single-phase implementation in underlying class
this.SpeciesIndexMapping[jagg] = null;
this.NoOfSpecies[jagg] = 1;
//this.XCompositeBasis[jagg] = null;
this.AggCellsSpecies[jagg] = null;
//this.CoarseToFineSpeciesIndex[jagg] = null;
}
else
{
//LsTrk.ContainesSpecies
int w = 0; // counter for found species
if (this.SpeciesIndexMapping[jagg] == null || this.SpeciesIndexMapping[jagg].GetLength(0) != w)
{
this.SpeciesIndexMapping[jagg] = new int[0, compCell.Length];
}
ArrayTools.SetAll(this.SpeciesIndexMapping[jagg], -897645);
// collect all species which are present in the composite cell 'jagg':
// -------------------------------------------------------------------
for (int k = 0; k < K; k++) // loop over all base cells in the aggregate cell 'jagg'
{
var rrc = _RRcs[k];
int jCell = compCell[k]; // base cell index
for (int iSpc = _NoOfSpecies[k] - 1; iSpc >= 0; iSpc--)
{
SpeciesId spId = LsTrk.GetSpeciesIdFromIndex(rrc, iSpc);
bool isPresent = LsTrk.Regions.IsSpeciesPresentInCell(spId, jCell);
bool isAgglomerated = agglomeratedCells.ContainsKey(spId) ? agglomeratedCells[spId][jCell] : false;
bool isUsed = Array.IndexOf(this.UsedSpecies, spId) >= 0;
if (isUsed && isPresent && !isAgglomerated)
{
bool bfound = false;
int z;
for (z = 0; z < w; z++) // loop over species found so far...
{
if (allPresentSpecies[z] == spId)
{
bfound = true;
break;
}
}
if (!bfound)
{
// species not found -> add to list
allPresentSpecies[w] = spId;
z = w;
w++;
}
// z is the species index in the composite cell
// iSpc is the species index in 'jCell'
Resize2DArray(ref this.SpeciesIndexMapping[jagg], w, K, -897645);
this.SpeciesIndexMapping[jagg][z, k] = iSpc;
}
}
}
this.AggCellsSpecies[jagg] = allPresentSpecies.GetSubVector(0, w);
this.NoOfSpecies[jagg] = w;
Debug.Assert(this.NoOfSpecies[jagg] == this.SpeciesIndexMapping[jagg].GetLength(0));
}
Debug.Assert(this.NoOfSpecies[jagg] >= 0);
Debug.Assert(this.NoOfSpecies[jagg] <= this.UsedSpecies.Length);
}
// MPI exchange for external cells
// note: external aggregation cells may be incomplete, i.e. not have all parts, only those known to parent
NoOfSpecies.MPIExchange(this.AggGrid);
}
protected override double RunSolverOneStep(int TimestepNo, double phystime, double dt)
{
Console.WriteLine(" Timestep # " + TimestepNo + ", phystime = " + phystime);
//phystime = 1.8;
LsUpdate(phystime);
// operator-matrix assemblieren
MsrMatrix OperatorMatrix = new MsrMatrix(u.Mapping, u.Mapping);
double[] Affine = new double[OperatorMatrix.RowPartitioning.LocalLength];
// Agglomerator setup
MultiphaseCellAgglomerator Agg = LsTrk.GetAgglomerator(new SpeciesId[] { LsTrk.GetSpeciesId("B") }, QuadOrder, this.THRESHOLD);
// plausibility of cell length scales
if (SER_PAR_COMPARISON)
{
TestLengthScales(QuadOrder, TimestepNo);
}
Console.WriteLine("Inter-Process agglomeration? " + Agg.GetAgglomerator(LsTrk.GetSpeciesId("B")).AggInfo.InterProcessAgglomeration);
if (this.THRESHOLD > 0.01)
{
TestAgglomeration_Extraploation(Agg);
TestAgglomeration_Projection(QuadOrder, Agg);
}
CheckExchange(true);
CheckExchange(false);
// operator matrix assembly
XSpatialOperatorMk2.XEvaluatorLinear mtxBuilder = Op.GetMatrixBuilder(base.LsTrk, u.Mapping, null, u.Mapping);
mtxBuilder.time = 0.0;
mtxBuilder.ComputeMatrix(OperatorMatrix, Affine);
Agg.ManipulateMatrixAndRHS(OperatorMatrix, Affine, u.Mapping, u.Mapping);
// mass matrix factory
var Mfact = LsTrk.GetXDGSpaceMetrics(new SpeciesId[] { LsTrk.GetSpeciesId("B") }, QuadOrder, 1).MassMatrixFactory;// new MassMatrixFactory(u.Basis, Agg);
// Mass matrix/Inverse Mass matrix
//var MassInv = Mfact.GetMassMatrix(u.Mapping, new double[] { 1.0 }, true, LsTrk.GetSpeciesId("B"));
var Mass = Mfact.GetMassMatrix(u.Mapping, new double[] { 1.0 }, false, LsTrk.GetSpeciesId("B"));
Agg.ManipulateMatrixAndRHS(Mass, default(double[]), u.Mapping, u.Mapping);
var MassInv = Mass.InvertBlocks(OnlyDiagonal: true, Subblocks: true, ignoreEmptyBlocks: true, SymmetricalInversion: false);
// test that operator depends only on B-species values
double DepTest = LsTrk.Regions.GetSpeciesSubGrid("B").TestMatrixDependency(OperatorMatrix, u.Mapping, u.Mapping);
Console.WriteLine("Matrix dependency test: " + DepTest);
Assert.LessOrEqual(DepTest, 0.0);
// diagnostic output
Console.WriteLine("Number of Agglomerations (all species): " + Agg.TotalNumberOfAgglomerations);
Console.WriteLine("Number of Agglomerations (species 'B'): " + Agg.GetAgglomerator(LsTrk.GetSpeciesId("B")).AggInfo.SourceCells.NoOfItemsLocally.MPISum());
// operator auswerten:
double[] x = new double[Affine.Length];
BLAS.daxpy(x.Length, 1.0, Affine, 1, x, 1);
OperatorMatrix.SpMVpara(1.0, u.CoordinateVector, 1.0, x);
MassInv.SpMV(1.0, x, 0.0, du_dx.CoordinateVector);
Agg.GetAgglomerator(LsTrk.GetSpeciesId("B")).Extrapolate(du_dx.Mapping);
// markieren, wo ueberhaupt A und B sind
Bmarker.AccConstant(1.0, LsTrk.Regions.GetSpeciesSubGrid("B").VolumeMask);
Amarker.AccConstant(+1.0, LsTrk.Regions.GetSpeciesSubGrid("A").VolumeMask);
if (usePhi0 && usePhi1)
{
Xmarker.AccConstant(+1.0, LsTrk.Regions.GetSpeciesSubGrid("X").VolumeMask);
}
// compute error
ERR.Clear();
ERR.Acc(1.0, du_dx_Exact, LsTrk.Regions.GetSpeciesSubGrid("B").VolumeMask);
ERR.Acc(-1.0, du_dx, LsTrk.Regions.GetSpeciesSubGrid("B").VolumeMask);
double L2Err = ERR.L2Norm(LsTrk.Regions.GetSpeciesSubGrid("B").VolumeMask);
Console.WriteLine("L2 Error: " + L2Err);
XERR.Clear();
XERR.GetSpeciesShadowField("B").Acc(1.0, ERR, LsTrk.Regions.GetSpeciesSubGrid("B").VolumeMask);
double xL2Err = XERR.L2Norm();
Console.WriteLine("L2 Error (in XDG space): " + xL2Err);
// check error
double ErrorThreshold = 1.0e-1;
if (this.MomentFittingVariant == XQuadFactoryHelper.MomentFittingVariants.OneStepGaussAndStokes)
{
ErrorThreshold = 1.0e-6; // HMF is designed for such integrands and should perform close to machine accuracy; on general integrands, the precision is different.
}
bool IsPassed = ((L2Err <= ErrorThreshold || this.THRESHOLD <= ErrorThreshold) && xL2Err <= ErrorThreshold);
if (IsPassed)
{
Console.WriteLine("Test PASSED");
}
else
{
Console.WriteLine("Test FAILED: check errors.");
//PlotCurrentState(phystime, TimestepNo, 3);
}
if (TimestepNo > 1)
{
if (this.THRESHOLD > ErrorThreshold)
{
// without agglomeration, the error in very tiny cut-cells may be large over the whole cell
// However, the error in the XDG-space should be small under all circumstances
Assert.LessOrEqual(L2Err, ErrorThreshold, "DG L2 error of computing du_dx");
}
Assert.LessOrEqual(xL2Err, ErrorThreshold, "XDG L2 error of computing du_dx");
}
// return/Ende
base.NoOfTimesteps = 17;
//base.NoOfTimesteps = 2;
dt = 0.3;
return(dt);
}
private void AssembleMatrix(double MU_A, double MU_B, out BlockMsrMatrix M, out double[] b, out MultiphaseCellAgglomerator agg, out MassMatrixFactory massFact)
{
using (var tr = new FuncTrace()) {
// create operator
// ===============
if (this.Control.SetDefaultDiriBndCnd)
{
this.Control.xLaplaceBCs.g_Diri = ((CommonParamsBnd inp) => 0.0);
this.Control.xLaplaceBCs.IsDirichlet = (inp => true);
}
double D = this.GridData.SpatialDimension;
int p = u.Basis.Degree;
double penalty_base = (p + 1) * (p + D) / D;
double penalty_multiplyer = base.Control.penalty_multiplyer;
XQuadFactoryHelper.MomentFittingVariants momentFittingVariant;
if (D == 3)
{
momentFittingVariant = XQuadFactoryHelper.MomentFittingVariants.Classic;
}
momentFittingVariant = this.Control.CutCellQuadratureType;
int order = this.u.Basis.Degree * 2;
XSpatialOperator Op = new XSpatialOperator(1, 1, (A, B, C) => order, "u", "c1");
var lengthScales = ((BoSSS.Foundation.Grid.Classic.GridData)GridData).Cells.PenaltyLengthScales;
var lap = new XLaplace_Bulk(this.LsTrk, penalty_multiplyer * penalty_base, "u", this.Control.xLaplaceBCs, 1.0, MU_A, MU_B, lengthScales, this.Control.ViscosityMode);
Op.EquationComponents["c1"].Add(lap); // Bulk form
Op.EquationComponents["c1"].Add(new XLaplace_Interface(this.LsTrk, MU_A, MU_B, penalty_base * 2, this.Control.ViscosityMode)); // coupling form
Op.Commit();
// create agglomeration
// ====================
var map = new UnsetteledCoordinateMapping(u.Basis);
//agg = new MultiphaseCellAgglomerator(
// new CutCellMetrics(momentFittingVariant,
// QuadOrderFunc.SumOfMaxDegrees(RoundUp: true)(map.BasisS.Select(bs => bs.Degree).ToArray(), new int[0], map.BasisS.Select(bs => bs.Degree).ToArray()),
// //this.HMFDegree,
// LsTrk, this.LsTrk.SpeciesIdS.ToArray()),
// this.Control.AgglomerationThreshold, false);
agg = LsTrk.GetAgglomerator(this.LsTrk.SpeciesIdS.ToArray(), order, this.Control.AgglomerationThreshold);
// compute matrix
// =============
using (new BlockTrace("XdgMatrixAssembly", tr)) {
M = new BlockMsrMatrix(map, map);
b = new double[M.RowPartitioning.LocalLength];
Op.ComputeMatrixEx(LsTrk,
map, null, map,
M, b, false, 0.0, true,
agg.CellLengthScales, null, null, //out massFact,
this.LsTrk.SpeciesIdS.ToArray());
}
// compare with linear evaluation
// ==============================
DGField[] testDomainFieldS = map.BasisS.Select(bb => new XDGField(bb as XDGBasis)).ToArray();
CoordinateVector test = new CoordinateVector(testDomainFieldS);
DGField[] errFieldS = map.BasisS.Select(bb => new XDGField(bb as XDGBasis)).ToArray();
CoordinateVector Err = new CoordinateVector(errFieldS);
var eval = Op.GetEvaluatorEx(LsTrk,
testDomainFieldS, null, map);
foreach (var s in this.LsTrk.SpeciesIdS)
{
eval.SpeciesOperatorCoefficients[s].CellLengthScales = agg.CellLengthScales[s];
}
eval.time = 0.0;
int L = test.Count;
Random r = new Random();
for (int i = 0; i < L; i++)
{
test[i] = r.NextDouble();
}
double[] R1 = new double[L];
double[] R2 = new double[L];
eval.Evaluate(1.0, 1.0, R1);
R2.AccV(1.0, b);
M.SpMV(1.0, test, 1.0, R2);
Err.AccV(+1.0, R1);
Err.AccV(-1.0, R2);
double ErrDist = GenericBlas.L2DistPow2(R1, R2).MPISum().Sqrt();
double Ref = test.L2NormPow2().MPISum().Sqrt();
Assert.LessOrEqual(ErrDist, Ref * 1.0e-5, "Mismatch between explicit evaluation of XDG operator and matrix.");
// agglomeration wahnsinn
// ======================
agg.ManipulateMatrixAndRHS(M, b, map, map);
foreach (var S in this.LsTrk.SpeciesNames)
{
Console.WriteLine(" Species {0}: no of agglomerated cells: {1}",
S, agg.GetAgglomerator(this.LsTrk.GetSpeciesId(S)).AggInfo.SourceCells.NoOfItemsLocally);
}
// mass matrix factory
// ===================
Basis maxB = map.BasisS.ElementAtMax(bss => bss.Degree);
//massFact = new MassMatrixFactory(maxB, agg);
massFact = LsTrk.GetXDGSpaceMetrics(this.LsTrk.SpeciesIdS.ToArray(), order).MassMatrixFactory;
}
}
/// <summary>
/// Mainly, comparison of single-core vs MPI-parallel run
/// </summary>
void TestLengthScales(int quadOrder, int TimestepNo)
{
string name_disc = $"t{TimestepNo}-alpha{this.THRESHOLD}-p{this.DEGREE}-q{MomentFittingVariant}";
var spcA = LsTrk.GetSpeciesId("A");
var spcB = LsTrk.GetSpeciesId("B");
var species = new[] { spcA, spcB };
//MultiphaseCellAgglomerator.CheckFile = $"InsideMpagg-{name_disc}.csv";
MultiphaseCellAgglomerator Agg = LsTrk.GetAgglomerator(species, quadOrder, this.THRESHOLD);
int RefMPIsize = 1;
// check level-set coordinates
// ===========================
{
var LsChecker = new TestingIO(this.GridData, $"LevelSets-{name_disc}.csv", RefMPIsize);
LsChecker.AddDGField(this.Phi0);
LsChecker.AddDGField(this.Phi1);
LsChecker.DoIOnow();
Assert.Less(LsChecker.AbsError(this.Phi0), 1.0e-8, "Mismatch in level-set 0 between single-core and parallel run.");
Assert.Less(LsChecker.AbsError(this.Phi1), 1.0e-8, "Mismatch in level-set 1 between single-core and parallel run.");
}
// check equality of agglomeration
// ===============================
// Note: agglomeration in parallel and serial mode is not necessarily equal - but very often it is.
// Therefore, we need to check whether the agglomeration is equal or not,
// in order to know whether agglomerated length scales should be compared or not.
bool[] equalAggAsinReferenceRun;
{
var aggoCheck = new TestingIO(this.GridData, $"Agglom-{name_disc}.csv", RefMPIsize);
long[] GiDs = GridData.CurrentGlobalIdPermutation.Values;
int J = GridData.iLogicalCells.NoOfLocalUpdatedCells;
long[] extGiDs = GridData.iParallel.GlobalIndicesExternalCells;
for (int iSpc = 0; iSpc < species.Length; iSpc++)
{
var spc = species[iSpc];
var spcN = LsTrk.GetSpeciesName(spc);
var ai = Agg.GetAgglomerator(spc).AggInfo;
var srcMask = ai.SourceCells.GetBitMask();
var aggPairs = ai.AgglomerationPairs;
aggoCheck.AddColumn($"SourceCells{spcN}", delegate(double[] X, int j, int jG) {
if (srcMask[j])
{
var pair = aggPairs.Single(cap => cap.jCellSource == j);
if (pair.jCellTarget < J)
{
return((double)GiDs[pair.jCellTarget]);
}
else
{
return((double)extGiDs[pair.jCellTarget - J]);
}
}
return(0.0);
});
}
aggoCheck.DoIOnow();
equalAggAsinReferenceRun = new bool[species.Length];
for (int iSpc = 0; iSpc < species.Length; iSpc++)
{
var spc = species[iSpc];
var spcN = LsTrk.GetSpeciesName(spc);
equalAggAsinReferenceRun[iSpc] = aggoCheck.AbsError($"SourceCells{spcN}") < 0.1;
if (equalAggAsinReferenceRun[iSpc] == false)
{
Console.WriteLine("Different agglomeration between single-core and parallel run for species " + spcN + ".");
}
}
//Agg.PlotAgglomerationPairs($"-aggPairs-{name_disc}-MPI{MPIRank + 1}of{MPISize}");
}
// compare length scales
// =====================
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out int rank);
foreach (ICutCellMetrics ccm in new ICutCellMetrics[] { Agg.NonAgglomeratedMetrics, Agg }) // loop over non-agglom and agglomerated metrics
{
string name;
bool Agglom;
if (object.ReferenceEquals(ccm, Agg))
{
name = "Agglomerated";
Agglom = true;
}
else
{
name = "Nonagglom";
Agglom = false;
}
MultidimensionalArray CellSurfaceA = ccm.CellSurface[spcA];
MultidimensionalArray CellVolumeA = ccm.CutCellVolumes[spcA];
MultidimensionalArray CellSurfaceB = ccm.CellSurface[spcB];
MultidimensionalArray CellVolumeB = ccm.CutCellVolumes[spcB];
string FileName = $"{name}LengthScales-{name_disc}.csv";
var Checker = new TestingIO(this.GridData, FileName, RefMPIsize);
Checker.AddColumn("CellSurfA", (double[] X, int j, int jG) => CellSurfaceA[j]);
Checker.AddColumn("CellVolA", (double[] X, int j, int jG) => CellVolumeA[j]);
Checker.AddColumn("CellSurfB", (double[] X, int j, int jG) => CellSurfaceB[j]);
Checker.AddColumn("CellVolB", (double[] X, int j, int jG) => CellVolumeB[j]);
Checker.DoIOnow();
if (this.MPISize == 1)
{
var Checker2 = new TestingIO(this.GridData, FileName, int.MaxValue);
Checker2.AddColumn("CellSurfA", (double[] X, int j, int jG) => CellSurfaceA[j]);
Checker2.AddColumn("CellVolA", (double[] X, int j, int jG) => CellVolumeA[j]);
Checker2.AddColumn("CellSurfB", (double[] X, int j, int jG) => CellSurfaceB[j]);
Checker2.AddColumn("CellVolB", (double[] X, int j, int jG) => CellVolumeB[j]);
Checker2.DoIOnow();
foreach (string s in Checker2.ColumnNames)
{
Assert.Less(Checker2.RelError(s), 1.0e-10, "'TestingUtils.Compare' itself is f****d up.");
}
}
double srfA = Checker.RelError("CellSurfA") * ((!Agglom || equalAggAsinReferenceRun[0]) ? 1.0 : 0.0);
double volA = Checker.RelError("CellVolA") * ((!Agglom || equalAggAsinReferenceRun[0]) ? 1.0 : 0.0);
double srfB = Checker.RelError("CellSurfB") * ((!Agglom || equalAggAsinReferenceRun[1]) ? 1.0 : 0.0);
double volB = Checker.RelError("CellVolB") * ((!Agglom || equalAggAsinReferenceRun[1]) ? 1.0 : 0.0);
if (srfA + volA + srfB + volB > 0.001)
{
Console.WriteLine($"Mismatch in {name} cell surface for species A between single-core and parallel run: {srfA}.");
Console.WriteLine($"Mismatch in {name} cell volume for species A between single-core and parallel run: {volA}.");
Console.WriteLine($"Mismatch in {name} cell surface for species B between single-core and parallel run: {srfB}.");
Console.WriteLine($"Mismatch in {name} cell volume for species B between single-core and parallel run: {volB}.");
}
Assert.Less(srfA, BLAS.MachineEps.Sqrt(), $"Mismatch in {name} cell surface for species A between single-core and parallel run.");
Assert.Less(volA, BLAS.MachineEps.Sqrt(), $"Mismatch in {name} cell volume for species A between single-core and parallel run.");
Assert.Less(srfB, BLAS.MachineEps.Sqrt(), $"Mismatch in {name} cell surface for species B between single-core and parallel run.");
Assert.Less(volB, BLAS.MachineEps.Sqrt(), $"Mismatch in {name} cell volume for species B between single-core and parallel run.");
}
}
