/// <summary>
/// This call computes an integral measure which may depend on
/// this <see cref="DGField"/> an the given <paramref name="function"/>;
/// This is a collective call, it must be invoked by all
/// MPI processes within the communicator; internally, it invokes MPI_Allreduce;
/// </summary>
/// <param name="function"></param>
/// <param name="rule">
/// composite quadrature rule.
/// </param>
/// <param name="Map">
/// Arbiter mapping applied to the values of this field and
/// <paramref name="function"/> at some point, which is finally integrated.
/// E.g., the mapping for an L2-error would be \f$ (\vec{x},a,b) => (a - b)^2 \f$,
/// where \f$ a \f$ is the value of this field at some point \f$ \vec{x} \f$ and
/// \f$ b \f$ is the value of <paramref name="function"/> at \f$ \vec{x} \f$.
/// </param>
/// <returns>
/// on all invoking MPI processes, the L2 norm of
/// this field minus <paramref name="function"/>
/// </returns>
public double LxError(ScalarFunctionEx function, Func <double[], double, double, double> Map, ICompositeQuadRule <QuadRule> rule)
{
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
using (new FuncTrace()) {
LxNormQuadrature l2nq = new LxNormQuadrature(this, function, Map, rule);
l2nq.Execute();
double nrmtot = l2nq.LxNorm.MPISum();
return(nrmtot);
}
}
/// <summary>
/// Estimates the cost imbalance between all processes using the given
/// <paramref name="estimator"/>
/// </summary>
/// <param name="estimator"></param>
/// <returns>
/// A number between 0 and 1 which represents an estimate of the load
/// imbalance in percent
/// </returns>
public static double ImbalanceEstimate(this ICellCostEstimator estimator)
{
MPICollectiveWatchDog.Watch();
double localCost = estimator.EstimatedLocalCost;
double[] allCosts = localCost.MPIAllGather();
double minCost = allCosts.Min();
double maxCost = allCosts.Max();
double imbalance = (maxCost - minCost) / Math.Max(double.Epsilon, maxCost);
return(imbalance);
}
/// <summary>
/// computes the L2 - norm based on Parceval's equality
/// </summary>
/// <remarks>
/// exploiting Parcevals equality for orthonormal systems, this
/// function is (should be?) much faster than
/// <see cref="DGField.L2Norm(CellMask)"/>
/// </remarks>
public override double L2Norm(CellMask cm)
{
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
using (new FuncTrace()) {
var DGCoordinates = this.Coordinates;
double nrm2_2 = 0;
if (cm == null)
{
int J = GridDat.iLogicalCells.NoOfLocalUpdatedCells;
int N = this.m_Basis.MaximalLength;
for (int j = 0; j < J; j++)
{
for (int n = 0; n < N; n++)
{
double k = DGCoordinates[j, n];
nrm2_2 += k * k;
}
}
}
else
{
int N = this.m_Basis.MaximalLength;
foreach (var c in cm)
{
int JE = c.Len + c.i0;
for (int j = c.i0; j < JE; j++)
{
for (int n = 0; n < N; n++)
{
double k = DGCoordinates[j, n];
nrm2_2 += k * k;
}
}
}
}
double nrmtot = 0;
unsafe
{
csMPI.Raw.Allreduce((IntPtr)(&nrm2_2), (IntPtr)(&nrmtot), 1, csMPI.Raw._DATATYPE.DOUBLE, csMPI.Raw._OP.SUM, csMPI.Raw._COMM.WORLD);
}
return(Math.Sqrt(nrmtot));
}
}
/// <summary>
/// Equivalent to <see cref="GetSpeciesSubGrid(string)"/>
/// </summary>
public SubGrid GetSpeciesSubGrid(SpeciesId specId)
{
MPICollectiveWatchDog.Watch();
if (m_SpeciesSubGrids == null)
{
m_SpeciesSubGrids = new Dictionary <SpeciesId, SubGrid>();
}
if (!m_SpeciesSubGrids.ContainsKey(specId))
{
CellMask cm = GetSpeciesMask(specId);
m_SpeciesSubGrids.Add(specId, new SubGrid(cm));
}
return(this.m_SpeciesSubGrids[specId]);
}
/// <summary>
/// Computes <see cref="SubgridIndex2LocalCellIndex"/>
/// </summary>
/// <returns></returns>
private int[] ComputeSubgridIndex2LocalCellIndex()
{
MPICollectiveWatchDog.Watch();
CellMask msk = m_VolumeMask;
int[] subgridIndex2LocalCellIndex = new int[msk.NoOfItemsLocally_WithExternal];
int jj = 0;
foreach (Chunk c in msk.GetEnumerableWithExternal())
{
for (int k = 0; k < c.Len; k++)
{
subgridIndex2LocalCellIndex[jj] = c.i0 + k;
jj++;
}
}
return(subgridIndex2LocalCellIndex);
}
/// <summary>
/// returns a bitmask that contains also information about external/ghost cells.
/// </summary>
public BitArray GetBitMaskWithExternal()
{
if (base.MaskType != MaskType.Logical)
{
throw new NotSupportedException();
}
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
int JE = this.GridData.iLogicalCells.Count;
int J = this.GridData.iLogicalCells.NoOfLocalUpdatedCells;
int MpiSize = this.GridData.CellPartitioning.MpiSize;
//Debugger.Break();
if (MpiSize > 1)
{
// more than one MPI process
// +++++++++++++++++++++++++
if (m_BitMaskWithExternal == null)
{
m_BitMaskWithExternal = new BitArray(JE, false);
// inner cells
foreach (Chunk c in this)
{
for (int i = 0; i < c.Len; i++)
{
m_BitMaskWithExternal[i + c.i0] = true;
}
}
m_BitMaskWithExternal.MPIExchange(this.GridData);
}
return(m_BitMaskWithExternal);
}
else
{
// single - processor mode
// +++++++++++++++++++++++
return(base.GetBitMask());
}
}
/// <summary>
/// computes the inner product of fields <paramref name="a"/> and <paramref name="b"/>
/// </summary>
static public double InnerProduct(DGField a, DGField b, CellQuadratureScheme quadScheme = null)
{
using (new FuncTrace()) {
if (!Object.ReferenceEquals(a.GridDat, b.GridDat))
{
throw new ApplicationException("fields must be defined on the same grid.");
}
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
InnerProductQuadrature ipq = new InnerProductQuadrature(a, b, quadScheme, a.Basis.Degree + b.Basis.Degree);
ipq.Execute();
unsafe {
double innerProdTot = double.NaN;
double InnerProdLoc = ipq.m_InnerProd;
csMPI.Raw.Allreduce((IntPtr)(&InnerProdLoc), (IntPtr)(&innerProdTot), 1, csMPI.Raw._DATATYPE.DOUBLE, csMPI.Raw._OP.SUM, csMPI.Raw._COMM.WORLD);
return(innerProdTot);
}
}
}
/// <summary>
/// returns the subgrid containing all cells in which the sign of the level set function #<paramref name="LevelSetIndex"/>
/// is at least in one point equal to <paramref name="sign"/>.
/// </summary>
public SubGrid GetLevelSetWing(int LevelSetIndex, double sign)
{
MPICollectiveWatchDog.Watch();
if (sign == 0.0)
{
throw new ArgumentException("must be either positive or negative");
}
if (LevelSetIndex < 0 || LevelSetIndex >= m_owner.m_LevelSetHistories.Count)
{
throw new IndexOutOfRangeException("invalid level set index");
}
int _sign = Math.Sign(sign);
int iwing = _sign * (LevelSetIndex + 1);
if (m_LevelSetWings == null)
{
m_LevelSetWings = new SortedDictionary <int, SubGrid>();
}
if (!m_LevelSetWings.ContainsKey(iwing))
{
int J = m_owner.GridDat.Cells.NoOfLocalUpdatedCells;
BitArray mask = new BitArray(J);
for (int j = 0; j < J; j++)
{
int dist = DecodeLevelSetDist(m_LevSetRegions[j], LevelSetIndex);
mask[j] = (dist * _sign >= 0);
}
SubGrid LevelSetWing = new SubGrid(new CellMask(m_owner.GridDat, mask));
m_LevelSetWings.Add(iwing, LevelSetWing);
}
return(m_LevelSetWings[iwing]);
}
/// <summary>
/// accumulates the derivative of DG field <paramref name="f"/>
/// (along the <paramref name="d"/>-th axis) times <paramref name="alpha"/>
/// to this field, i.e. <br/>
/// this = this + <paramref name="alpha"/>* \f$ \frac{\partial}{\partial x_d} \f$ <paramref name="f"/>;
/// </summary>
/// <param name="f"></param>
/// <param name="d">
/// 0 for the x-derivative, 1 for the y-derivative, 2 for the z-derivative
/// </param>
/// <param name="alpha">
/// scaling of <paramref name="f"/>;
/// </param>
/// <param name="em">
/// An optional restriction to the domain in which the derivative is computed (it may, e.g.
/// be only required in boundary cells, so a computation over the whole domain
/// would be a waste of computation power. A proper execution mask for this case would be e.g.
/// <see cref="BoSSS.Foundation.Grid.GridData.BoundaryCells"/>.)<br/>
/// if null, the computation is carried out in the whole domain
/// </param>
override public void Derivative(double alpha, DGField f, int d, CellMask em)
{
using (var tr = new FuncTrace()) {
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
if (this.Basis.Degree < f.Basis.Degree - 1)
{
throw new ArgumentException("cannot compute derivative because of incompatible basis functions.", "f");
}
if (f.Basis.GetType() != this.Basis.GetType())
{
throw new ArgumentException("cannot compute derivative because of incompatible basis functions.", "f");
}
int D = GridDat.SpatialDimension;
if (d < 0 || d >= D)
{
throw new ArgumentException("spatial dimension out of range.", "d");
}
Quadrature.EdgeQuadratureScheme _qInsEdge;
Quadrature.CellQuadratureScheme _qInsVol;
{
_qInsEdge = (new Quadrature.EdgeQuadratureScheme(false, EdgeMask.GetEmptyMask(this.GridDat)));
_qInsVol = (new Quadrature.CellQuadratureScheme(true, em));
}
var op = (new BrokenDerivativeForm(d)).Operator(1);
op.Evaluate(alpha, 1.0, f.Mapping, null, this.Mapping,
qInsEdge: _qInsEdge,
qInsVol: _qInsVol);
}
}
/// <summary>
/// ctor.
/// </summary>
internal XQuadSchemeHelper(XDGSpaceMetrics __XDGSpaceMetrics)
{
MPICollectiveWatchDog.Watch();
this.XDGSpaceMetrics = __XDGSpaceMetrics;
ConstructorCommon();
}
/// <summary>
/// creates a HYPRE solver
/// </summary>
public Solver()
{
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
CreateSolver();
}
/// <summary>
/// Returns the minimum and the maximum value of the DG field
/// This is a collective call, it must be invoked by all
/// MPI processes within the communicator; internally, it invokes MPI_Allreduce;
/// </summary>
/// <param name="max">
/// on all invoking MPI processes, the maximum value (over all processors) of
/// this field;
/// </param>
/// <param name="min">
/// on all invoking MPI processes, the minimum value (over all processors) of
/// this field;
/// </param>
/// <remarks>
/// to find the maximum value, this field is evaluated on each vertex and the center of the simplex.
/// </remarks>
/// <param name="cm">
/// optional domain restriction
/// </param>
/// <param name="jMaxGlob">
/// global cell index in which the maximum value <paramref name="max"/> was found
/// </param>
/// <param name="jMinGlob">
/// global cell index in which the maximum value <paramref name="min"/> was found
/// </param>
public void GetExtremalValues(out double min, out double max, out int jMinGlob, out int jMaxGlob, CellMask cm = null)
{
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
using (new FuncTrace()) {
int J = Basis.GridDat.iLogicalCells.NoOfLocalUpdatedCells;
// find maximum on this processor
// ------------------------------
// create node set
InitExtremalProbeNS();
// vectorisation of this method
int VectorSize = -1;
int N0 = m_ExtremalProbeNS[0].GetLength(0); // number of nodes
MultidimensionalArray fieldValues = new MultidimensionalArray(2);
IEnumerable <Chunk> all_chunks;
if (cm == null)
{
var _ch = new Chunk[1];
_ch[0].i0 = 0;
_ch[0].Len = J;
all_chunks = _ch;
}
else
{
all_chunks = cm;
}
double LocMax = -double.MaxValue, LocMin = double.MaxValue;
int jMinLoc = int.MaxValue, jMaxLoc = int.MaxValue;
foreach (Chunk chk in all_chunks)
{
VectorSize = this.GridDat.iGeomCells.GetNoOfSimilarConsecutiveCells(CellInfo.RefElementIndex_Mask, chk.i0, Math.Min(100, chk.Len)); // try to be a little vectorized;
int _J = chk.Len + chk.i0;
for (int j = chk.i0; j < _J; j += VectorSize)
{
if (j + VectorSize > _J)
{
VectorSize = _J - j;
}
int iKref = Basis.GridDat.iGeomCells.GetRefElementIndex(j);
int N = m_ExtremalProbeNS[iKref].GetLength(0);
if (fieldValues.GetLength(0) != VectorSize || fieldValues.GetLength(1) != N)
{
fieldValues = MultidimensionalArray.Create(VectorSize, N);
}
this.Evaluate(j, VectorSize, m_ExtremalProbeNS[iKref], fieldValues, 0.0);
// loop over cells ...
for (int jj = j; jj < j + VectorSize; jj++)
{
// loop over nodes ...
for (int n = 0; n < N; n++)
{
double vel = 0;
vel = fieldValues[jj - j, n];
if (vel > LocMax)
{
LocMax = vel;
jMaxLoc = jj;
}
if (vel < LocMin)
{
LocMin = vel;
jMinLoc = jj;
}
}
}
}
}
// find the maximum over all processes via MPI and return
// ------------------------------------------------------
if (Basis.GridDat.CellPartitioning.MpiSize > 1)
{
double[] total = new double[2];
double[] local = new double[] { LocMax, -LocMin };
unsafe
{
fixed(double *ploc = local, ptot = total)
{
csMPI.Raw.Allreduce((IntPtr)ploc, (IntPtr)ptot, 2, csMPI.Raw._DATATYPE.DOUBLE, csMPI.Raw._OP.MAX, csMPI.Raw._COMM.WORLD);
}
}
max = total[0];
min = -total[1];
int i0 = Basis.GridDat.CellPartitioning.i0;
int[] jGlob = new int[2];
if (max == LocMax)
{
// (at least one) global maximum on this processor
jGlob[0] = jMaxLoc + i0;
}
else
{
// maximum on some other processor
jGlob[0] = int.MaxValue;
}
if (min == LocMin)
{
// (at least one) global minimum on this processor
jGlob[1] = jMinLoc + i0;
}
else
{
// minimum on some other processor
jGlob[1] = int.MaxValue;
}
// in case of multiple global minimums/maximums, e.g. for a constant field, we return the lowest (jMaxGlob,jMinGlob)
int[] jGlobM = new int[2];
unsafe
{
fixed(int *ploc = jGlob, ptot = jGlobM)
{
csMPI.Raw.Allreduce((IntPtr)ploc, (IntPtr)ptot, 2, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.MIN, csMPI.Raw._COMM.WORLD);
}
}
jMaxGlob = jGlob[0];
jMinGlob = jGlob[1];
}
else
{
min = LocMin;
max = LocMax;
jMaxGlob = jMaxLoc;
jMinGlob = jMinLoc;
}
}
}
/// <summary>
/// Computes Cell-volumes and edge areas before agglomeration.
/// </summary>
void ComputeNonAgglomeratedMetrics()
{
using (new FuncTrace()) {
MPICollectiveWatchDog.Watch();
var gd = XDGSpaceMetrics.GridDat;
int JE = gd.iLogicalCells.Count;
int J = gd.iLogicalCells.NoOfLocalUpdatedCells;
int D = gd.SpatialDimension;
int EE = gd.Edges.Count;
SpeciesId[] species = this.SpeciesList.ToArray();
int NoSpc = species.Count();
int[,] E2C = gd.iLogicalEdges.CellIndices;
var schH = new XQuadSchemeHelper(XDGSpaceMetrics);
// collect all per-cell-metrics in the same MultidimArry, for MPI-exchange (only 1 exchange instead of three, saving some overhead)
// 1st index: cell
// 2nd index: species
// 3rd index: 0 for interface surface per cell, 1 for cut-cell-volume, 2 for cut-cell surface
double[] vec_cellMetrics = new double[JE * NoSpc * 3];
MultidimensionalArray cellMetrics = MultidimensionalArray.CreateWrapper(vec_cellMetrics, JE, NoSpc, 3);
this.CutEdgeAreas = new Dictionary <SpeciesId, MultidimensionalArray>();
this.CutCellVolumes = new Dictionary <SpeciesId, MultidimensionalArray>();
this.InterfaceArea = new Dictionary <SpeciesId, MultidimensionalArray>();
this.CellSurface = new Dictionary <SpeciesId, MultidimensionalArray>();
//var schS = new List<CellQuadratureScheme>();
//var rulz = new List<ICompositeQuadRule<QuadRule>>();
// edges and volumes
// =================
for (int iSpc = 0; iSpc < species.Length; iSpc++)
{
var cellVol = cellMetrics.ExtractSubArrayShallow(-1, iSpc, 1);
var spc = species[iSpc];
// compute cut edge area
// ---------------------
MultidimensionalArray edgArea = MultidimensionalArray.Create(EE);
this.CutEdgeAreas.Add(spc, edgArea);
var edgeScheme = schH.GetEdgeQuadScheme(spc);
var edgeRule = edgeScheme.Compile(gd, this.CutCellQuadratureOrder);
BoSSS.Foundation.Quadrature.EdgeQuadrature.GetQuadrature(
new int[] { 1 }, gd,
edgeRule,
_Evaluate : delegate(int i0, int Length, QuadRule QR, MultidimensionalArray EvalResult) //
{
EvalResult.SetAll(1.0);
},
_SaveIntegrationResults : delegate(int i0, int Length, MultidimensionalArray ResultsOfIntegration) //
{
for (int i = 0; i < Length; i++)
{
int iEdge = i + i0;
Debug.Assert(edgArea[iEdge] == 0);
edgArea[iEdge] = ResultsOfIntegration[i, 0];
Debug.Assert(!(double.IsNaN(edgArea[iEdge]) || double.IsInfinity(edgArea[iEdge])));
}
}).Execute();
// sum up edges for surface
// ------------------------
var cellSurf = cellMetrics.ExtractSubArrayShallow(-1, iSpc, 2);
for (int e = 0; e < EE; e++)
{
double a = edgArea[e];
int jCell0 = E2C[e, 0];
int jCell2 = E2C[e, 1];
cellSurf[jCell0] += a;
if (jCell2 >= 0)
{
cellSurf[jCell2] += a;
}
}
// compute cut cell volumes
// ------------------------
var volScheme = schH.GetVolumeQuadScheme(spc);
var volRule = volScheme.Compile(gd, this.CutCellQuadratureOrder);
BoSSS.Foundation.Quadrature.CellQuadrature.GetQuadrature(
new int[] { 1 }, gd,
volRule,
_Evaluate : delegate(int i0, int Length, QuadRule QR, MultidimensionalArray EvalResult) //
{
EvalResult.SetAll(1.0);
},
_SaveIntegrationResults : delegate(int i0, int Length, MultidimensionalArray ResultsOfIntegration) //
{
for (int i = 0; i < Length; i++)
{
int jCell = i + i0;
Debug.Assert(cellVol[jCell] == 0);
cellVol[jCell] = ResultsOfIntegration[i, 0];
Debug.Assert(!(double.IsNaN(cellVol[jCell]) || double.IsInfinity(cellVol[jCell])));
}
}).Execute();
}
// interface surface
// =================
// loop over level-sets
if (species.Length > 0)
{
// find domain of all species:
CellMask SpeciesCommonDom = XDGSpaceMetrics.LevelSetRegions.GetSpeciesMask(species[0]);
for (int iSpc = 1; iSpc < species.Length; iSpc++)
{
SpeciesCommonDom = SpeciesCommonDom.Union(XDGSpaceMetrics.LevelSetRegions.GetSpeciesMask(species[iSpc]));
}
BitArray[] SpeciesBitMask = species.Select(spc => XDGSpaceMetrics.LevelSetRegions.GetSpeciesMask(spc).GetBitMask()).ToArray();
int NoOfLs = XDGSpaceMetrics.NoOfLevelSets;
int NoOfSpc = species.Length;
for (int iLevSet = 0; iLevSet < NoOfLs; iLevSet++)
{
var LsDom = XDGSpaceMetrics.LevelSetRegions.GetCutCellMask4LevSet(iLevSet);
var IntegrationDom = LsDom.Intersect(SpeciesCommonDom);
//if (IntegrationDom.NoOfItemsLocally > 0) { -> Doesn't work if Bjoerns HMF is used, eds up in an mpi dead lock
// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
// this level-set is actually relevant for someone in 'species'
// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
CellQuadratureScheme SurfIntegration = schH.GetLevelSetquadScheme(iLevSet, IntegrationDom);
var rule = SurfIntegration.Compile(gd, this.CutCellQuadratureOrder);
BoSSS.Foundation.Quadrature.CellQuadrature.GetQuadrature(
new int[] { 1 }, gd,
rule,
_Evaluate : delegate(int i0, int Length, QuadRule QR, MultidimensionalArray EvalResult) {
EvalResult.SetAll(1.0);
},
_SaveIntegrationResults : delegate(int i0, int Length, MultidimensionalArray ResultsOfIntegration) {
for (int i = 0; i < Length; i++)
{
int jCell = i + i0;
//if (cellMetrics[jCell, iSpcA, 0] != 0.0)
// throw new NotSupportedException("More than one zero-level-set per cell is not supported yet.");
//if (cellMetrics[jCell, iSpcB, 0] != 0.0)
// throw new NotSupportedException("More than one zero-level-set per cell is not supported yet.");
//cellMetrics[jCell, iSpcA, 0] = ResultsOfIntegration[i, 0];
//cellMetrics[jCell, iSpcB, 0] = ResultsOfIntegration[i, 0];
for (int iSpc = 0; iSpc < NoOfSpc; iSpc++)
{
if (SpeciesBitMask[iSpc][jCell])
{
cellMetrics[jCell, iSpc, 0] += ResultsOfIntegration[i, 0];
Debug.Assert(!(double.IsNaN(cellMetrics[jCell, iSpc, 0]) || double.IsInfinity(cellMetrics[jCell, iSpc, 0])));
}
}
}
}).Execute();
//}
}
}
// MPI exchange & store
// ====================
if (species.Length > 0)
{
#if DEBUG
int NoOfSpc = species.Length;
var cellMetricsB4 = cellMetrics.ExtractSubArrayShallow(new[] { 0, 0, 0 }, new[] { J - 1, NoOfSpc - 1, 1 }).CloneAs();
#endif
vec_cellMetrics.MPIExchange(gd);
#if DEBUG
var cellMetricsComp = cellMetrics.ExtractSubArrayShallow(new[] { 0, 0, 0 }, new[] { J - 1, NoOfSpc - 1, 1 }).CloneAs();
cellMetricsComp.Acc(-1.0, cellMetricsB4);
Debug.Assert(cellMetricsComp.L2Norm() == 0.0);
#endif
}
for (int iSpc = 0; iSpc < species.Length; iSpc++)
{
var spc = species[iSpc];
this.InterfaceArea.Add(spc, cellMetrics.ExtractSubArrayShallow(-1, iSpc, 0).CloneAs());
this.CutCellVolumes.Add(spc, cellMetrics.ExtractSubArrayShallow(-1, iSpc, 1).CloneAs());
this.CellSurface.Add(spc, cellMetrics.ExtractSubArrayShallow(-1, iSpc, 2).CloneAs());
this.CellSurface[spc].Acc(1.0, this.InterfaceArea[spc]);
}
}
}
/// <summary>
/// Polynomial extrapolation between cells.
/// </summary>
/// <param name="ExtrapolateTo">contains all cells for which extrapolated values should be computed</param>
/// <param name="ExtrapolateFrom">contains all cells with "known values"</param>
/// <returns>
/// The number of cells (locally) for which the algorithm was not able to extrapolate a value
/// </returns>
virtual public int CellExtrapolation(CellMask ExtrapolateTo, CellMask ExtrapolateFrom)
{
MPICollectiveWatchDog.Watch();
int J = this.GridDat.iLogicalCells.NoOfLocalUpdatedCells;
int NoOfNeigh;
int[] NeighIdx;
int[][] CN = this.GridDat.iLogicalCells.CellNeighbours;
// mark all cells in which species 'Id' is known
// ---------------------------------------------
BitArray ValueIsKnown = ExtrapolateFrom.GetBitMaskWithExternal().CloneAs();
CellMask _ExtrapolateTo = ExtrapolateTo.Except(ExtrapolateFrom);
BitArray NeedToKnowValue = _ExtrapolateTo.GetBitMask();
this.Clear(_ExtrapolateTo);
// repeat until (for species 'Id' the DOF' of) all cells
// that contain (at least a fraction of) species 'Id' are known...
// ------------------------------------------------------------------
int NoOfCells_ToExtrapolate = _ExtrapolateTo.NoOfItemsLocally;
int NoOfCells_ExtrapolatedInSweep = 1;
int sweepcnt = 0;
List <double> scaling = new List <double>();
List <int[]> CellPairs = new List <int[]>();
BitArray cells_mod_in_sweep = new BitArray(J);
while (true)
{
// MPI-parallel evaluation of termination criterion
// ------------------------------------------------
int bool_LocalRun = ((NoOfCells_ToExtrapolate > 0) && (NoOfCells_ExtrapolatedInSweep > 0)) ? 1 : 0;
int bool_GlobalRun = 0;
unsafe
{
csMPI.Raw.Allreduce((IntPtr)(&bool_LocalRun), (IntPtr)(&bool_GlobalRun), 1, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.MAX, csMPI.Raw._COMM.WORLD);
}
if (bool_GlobalRun <= 0)
{
// finished on all MPI processors
break;
}
// MPI-update before local sweep: necessary for consistent result of alg.
this.MPIExchange();
if (sweepcnt > 0)
{
ValueIsKnown.MPIExchange(this.GridDat);
}
// local work
// ----------
if (bool_LocalRun > 0)
{
sweepcnt++;
NoOfCells_ExtrapolatedInSweep = 0;
scaling.Clear();
CellPairs.Clear();
cells_mod_in_sweep.SetAll(false);
for (int j = 0; j < J; j++)
{
// determine whether there is something to do for cell 'j' or not ...
bool needToKnowSpecies = NeedToKnowValue[j];
bool _mustbeExtrapolated = needToKnowSpecies && !ValueIsKnown[j];
if (_mustbeExtrapolated)
{
// continuation for this cell is needed
// ++++++++++++++++++++++++++++++++++++
// try to find a neighbour
NeighIdx = CN[j].CloneAs();
NoOfNeigh = NeighIdx.Length;
int FoundNeighs = 0;
for (int nn = 0; nn < NoOfNeigh; nn++)
{
if (ValueIsKnown[NeighIdx[nn]])
{
// bingo
FoundNeighs++;
}
else
{
NeighIdx[nn] = -1; // not usable
}
}
if (FoundNeighs <= 0)
{
// hope for better luck in next sweep
continue;
}
//Array.Clear(u2, 0, N);
for (int nn = 0; nn < NoOfNeigh; nn++)
{
if (NeighIdx[nn] < 0)
{
continue;
}
int _2 = j; // cell to extrapolate TO
int _1 = NeighIdx[nn]; // cell to extrapolate FROM
CellPairs.Add(new int[] { _1, _2 });
double ooFoundNeighs = 1.0 / (double)FoundNeighs;
scaling.Add(ooFoundNeighs);
}
cells_mod_in_sweep[j] = true;
NoOfCells_ExtrapolatedInSweep++;
}
}
int E = CellPairs.Count;
int[,] _CellPairs = new int[E, 2];
for (int e = 0; e < E; e++)
{
_CellPairs.SetRow(e, CellPairs[e]);
}
double[] preScale = new double[scaling.Count];
preScale.SetAll(1.0);
this.CellExtrapolation(_CellPairs, scaling, preScale);
for (int j = 0; j < J; j++)
{
if (cells_mod_in_sweep[j] == true)
{
ValueIsKnown[j] = true;
}
}
NoOfCells_ToExtrapolate -= NoOfCells_ExtrapolatedInSweep;
}
}
// return
// ------
return(NoOfCells_ToExtrapolate);
}
/// <summary>
/// constructor
/// </summary>
protected Solver()
{
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
}
/// <summary>
/// ctor.
/// </summary>
internal NECQuadratureLevelSet(IGridData context,
XSpatialOperatorMk2 DiffOp,
V __ResultVector,
IList <DGField> __DomainFields,
IList <DGField> __Parameters,
UnsetteledCoordinateMapping CodomainMap,
LevelSetTracker lsTrk, int _iLevSet, Tuple <SpeciesId, SpeciesId> SpeciesPair,
ICompositeQuadRule <QuadRule> domAndRule) //
: base(new int[] { CodomainMap.GetNonXBasisLengths(0).Sum() * 2 }, // we always integrate over species in pairs (neg + pos), so we need to alloc mem only 2 species
context,
domAndRule) //
{
MPICollectiveWatchDog.Watch();
// -----------------------------------
// set members / check ctor parameters
// -----------------------------------
m_lsTrk = lsTrk;
this.m_LevSetIdx = _iLevSet;
this.m_SpeciesPair = SpeciesPair;
this.ResultVector = __ResultVector;
m_CodomainMap = CodomainMap;
var _Parameters = (__Parameters != null) ? __Parameters.ToArray() : new DGField[0];
if (__DomainFields.Count != DiffOp.DomainVar.Count)
{
throw new ArgumentException("mismatch between number of domain variables in spatial operator and given domain variables");
}
if (_Parameters.Length != DiffOp.ParameterVar.Count)
{
throw new ArgumentException("mismatch between number of parameter variables in spatial operator and given parameters");
}
if (m_CodomainMap.NoOfVariables != DiffOp.CodomainVar.Count)
{
throw new ArgumentException("mismatch between number of codomain variables in spatial operator and given codomain mapping");
}
var _DomainAndParamFields = ArrayTools.Cat(__DomainFields, _Parameters);
this.DELTA = __DomainFields.Count;
m_DomainAndParamFieldsA = new ConventionalDGField[_DomainAndParamFields.Length];
m_DomainAndParamFieldsB = new ConventionalDGField[_DomainAndParamFields.Length];
for (int i = 0; i < m_DomainAndParamFieldsA.Length; i++)
{
var f = _DomainAndParamFields[i];
if (f == null)
{
m_DomainAndParamFieldsA[i] = null;
m_DomainAndParamFieldsB[i] = null;
}
else if (f is XDGField xf)
{
m_DomainAndParamFieldsA[i] = xf.GetSpeciesShadowField(this.SpeciesA);
m_DomainAndParamFieldsB[i] = xf.GetSpeciesShadowField(this.SpeciesB);
}
else if (f is ConventionalDGField cf)
{
m_DomainAndParamFieldsA[i] = cf;
m_DomainAndParamFieldsB[i] = null;
}
else
{
throw new NotImplementedException("missing implementation for " + f.GetType().Name);
}
}
LECQuadratureLevelSet <IMutableMatrix, double[]> .TestNegativeAndPositiveSpecies(domAndRule, m_lsTrk, SpeciesA, SpeciesB, m_LevSetIdx);
// ------------------------
// sort equation components
// ------------------------
int Gamma = m_CodomainMap.NoOfVariables;
m_NonlinLsForm_V = EquationComponentArgMapping <INonlinLevelSetForm_V> .GetArgMapping(DiffOp, true,
eq => ((eq.LevelSetTerms & (TermActivationFlags.V | TermActivationFlags.UxV | TermActivationFlags.GradUxV)) != 0) && Compfilter(eq),
eq => (eq is ILevelSetForm)?new NonlinearLevelSetFormVectorizer((ILevelSetForm)eq, lsTrk) : null);
m_NonlinLsForm_GradV = EquationComponentArgMapping <INonlinLevelSetForm_GradV> .GetArgMapping(DiffOp, true,
eq => ((eq.LevelSetTerms & (TermActivationFlags.GradV | TermActivationFlags.UxGradV | TermActivationFlags.GradUxGradV)) != 0) && Compfilter(eq),
eq => (eq is ILevelSetForm)?new NonlinearLevelSetFormVectorizer((ILevelSetForm)eq, lsTrk) : null);
m_ValueRequired = new bool[m_DomainAndParamFieldsA.Length];
m_GradientRequired = new bool[m_DomainAndParamFieldsA.Length];
m_NonlinLsForm_V.DetermineReqFields(m_GradientRequired,
comp => ((comp.LevelSetTerms & (TermActivationFlags.GradUxGradV | TermActivationFlags.GradUxV)) != 0));
m_NonlinLsForm_GradV.DetermineReqFields(m_GradientRequired,
comp => ((comp.LevelSetTerms & (TermActivationFlags.GradUxGradV | TermActivationFlags.GradUxV)) != 0));
m_NonlinLsForm_V.DetermineReqFields(m_ValueRequired,
comp => ((comp.LevelSetTerms & (TermActivationFlags.UxGradV | TermActivationFlags.UxV)) != 0));
m_NonlinLsForm_GradV.DetermineReqFields(m_ValueRequired,
comp => ((comp.LevelSetTerms & (TermActivationFlags.UxGradV | TermActivationFlags.UxV)) != 0));
for (int i = __DomainFields.Count; i < m_DomainAndParamFieldsA.Length; i++)
{
m_ValueRequired[i] = true; // parameters are always required!
}
// -----
// alloc
// -----
Koeff_V = new MultidimensionalArray[Gamma];
Koeff_GradV = new MultidimensionalArray[Gamma];
for (int gamma = 0; gamma < Gamma; gamma++)
{
Koeff_V[gamma] = new MultidimensionalArray(3);
Koeff_GradV[gamma] = new MultidimensionalArray(4);
}
Debug.Assert(m_DomainAndParamFieldsA.Length == m_DomainAndParamFieldsB.Length);
int L = m_DomainAndParamFieldsA.Length;
m_FieldValuesPos = new MultidimensionalArray[L];
m_FieldValuesNeg = new MultidimensionalArray[L];
m_FieldGradientValuesPos = new MultidimensionalArray[L];
m_FieldGradientValuesNeg = new MultidimensionalArray[L];
for (int l = 0; l < L; l++)
{
if (m_ValueRequired[l])
{
m_FieldValuesPos[l] = new MultidimensionalArray(2);
m_FieldValuesNeg[l] = new MultidimensionalArray(2);
}
if (m_GradientRequired[l])
{
m_FieldGradientValuesPos[l] = new MultidimensionalArray(3);
m_FieldGradientValuesNeg[l] = new MultidimensionalArray(3);
}
}
// ------------------
// init custom timers
// ------------------
base.CustomTimers = new Stopwatch[] { new Stopwatch(), new Stopwatch(), new Stopwatch(), new Stopwatch(), new Stopwatch() };
base.CustomTimers_Names = new string[] { "Flux-Eval", "Basis-Eval", "Loops", "ParametersAndNormals", "Field-Eval" };
base.CustomTimers_RootPointer = new int[5];
ArrayTools.SetAll(base.CustomTimers_RootPointer, -1);
this.m_NonlinLsForm_V_Watches = this.m_NonlinLsForm_V.InitStopWatches(0, this);
this.m_NonlinLsForm_GradV_Watches = this.m_NonlinLsForm_GradV.InitStopWatches(0, this);
Flux_Eval = base.CustomTimers[0];
Basis_Eval = base.CustomTimers[1];
Loops = base.CustomTimers[2];
ParametersAndNormals = base.CustomTimers[3];
Field_Eval = base.CustomTimers[4];
}
//
//
/// <summary>
/// applies the agglomeration on a general matrix
/// </summary>
/// <param name="Matrix">the matrix that should be manipulated.</param>
/// <param name="Rhs">the right-hand-side that should be manipulated</param>
/// <param name="ColMap"></param>
/// <param name="ColMapAggSw">Turns column agglomeration on/off fore each variable individually; default == null is on. </param>
/// <param name="RowMap"></param>
/// <param name="RowMapAggSw">The same shit as for <paramref name="ColMapAggSw"/>, just for rows.</param>
public void ManipulateMatrixAndRHS<M, T>(M Matrix, T Rhs, UnsetteledCoordinateMapping RowMap, UnsetteledCoordinateMapping ColMap, bool[] RowMapAggSw = null, bool[] ColMapAggSw = null)
where M : IMutableMatrixEx
where T : IList<double>
{
MPICollectiveWatchDog.Watch();
//var mtxS = GetFrameMatrices(Matrix, RowMap, ColMap);
if (Matrix == null && Rhs == null)
// nothing to do
return;
if (TotalNumberOfAgglomerations <= 0)
// nothing to do
return;
if (RowMapAggSw != null)
throw new NotImplementedException();
// generate agglomeration sparse matrices
// ======================================
int RequireRight;
if (Matrix == null) {
// we don't need multiplication-from-the-right at all
RequireRight = 0;
} else {
if (RowMap.EqualsUnsetteled(ColMap) && ArrayTools.ListEquals(ColMapAggSw, RowMapAggSw)) {
// we can use the same matrix for right and left multiplication
RequireRight = 1;
} else {
// separate matrix for the multiplication-from-the-right is required
RequireRight = 2;
}
}
BlockMsrMatrix LeftMul = null, RightMul = null;
{
foreach (var kv in DictAgglomeration) {
var Species = kv.Key;
var m_Agglomerator = kv.Value;
if (m_Agglomerator != null) {
CellMask spcMask = this.Tracker.Regions.GetSpeciesMask(Species);
MiniMapping rowMini = new MiniMapping(RowMap, Species, this.Tracker.Regions);
BlockMsrMatrix LeftMul_Species = m_Agglomerator.GetRowManipulationMatrix(RowMap, rowMini.MaxDeg, rowMini.NoOfVars, rowMini.i0Func, rowMini.NFunc, false, spcMask);
if (LeftMul == null) {
LeftMul = LeftMul_Species;
} else {
LeftMul.Acc(1.0, LeftMul_Species);
}
if (!object.ReferenceEquals(LeftMul, RightMul) && RightMul != null) {
MiniMapping colMini = new MiniMapping(ColMap, Species, this.Tracker.Regions);
BlockMsrMatrix RightMul_Species = m_Agglomerator.GetRowManipulationMatrix(ColMap, colMini.MaxDeg, colMini.NoOfVars, colMini.i0Func, colMini.NFunc, false, spcMask);
if (RightMul == null) {
RightMul = RightMul_Species;
} else {
RightMul.Acc(1.0, RightMul_Species);
}
} else if (RequireRight == 1) {
RightMul = LeftMul;
} else {
RightMul = null;
}
}
}
}
// apply the agglomeration to the matrix
// =====================================
if (Matrix != null) {
BlockMsrMatrix RightMulTr = RightMul.Transpose();
BlockMsrMatrix _Matrix;
if (Matrix is BlockMsrMatrix) {
_Matrix = (BlockMsrMatrix)((object)Matrix);
} else {
_Matrix = Matrix.ToBlockMsrMatrix(RowMap, ColMap);
}
var AggMatrix = BlockMsrMatrix.Multiply(LeftMul, BlockMsrMatrix.Multiply(_Matrix, RightMulTr));
if (object.ReferenceEquals(_Matrix, Matrix)) {
_Matrix.Clear();
_Matrix.Acc(1.0, AggMatrix);
} else {
Matrix.Acc(-1.0, _Matrix); // das ist so
Matrix.Acc(1.0, AggMatrix); // meagaschlecht !!!!!!
}
}
// apply the agglomeration to the Rhs
// ==================================
if (Rhs != null) {
double[] tmp = Rhs.ToArray();
if (object.ReferenceEquals(tmp, Rhs))
throw new ApplicationException("Flache kopie sollte eigentlich ausgeschlossen sein!?");
LeftMul.SpMV(1.0, tmp, 0.0, Rhs);
}
}
//
public static IEnumerable<Tuple<int, int>> FindAgglomeration(LevelSetTracker Tracker, SpeciesId spId, double AgglomerationThreshold,
MultidimensionalArray CellVolumes, MultidimensionalArray edgeArea,
bool AgglomerateNewborn, bool AgglomerateDeceased, bool ExceptionOnFailedAgglomeration,
MultidimensionalArray[] oldCellVolumes, double[] oldTs__AgglomerationTreshold, double NewbornAndDecasedThreshold)
{
using (var tracer = new FuncTrace()) {
MPICollectiveWatchDog.Watch();
// init
// =======
GridData grdDat = Tracker.GridDat;
int[,] Edge2Cell = grdDat.Edges.CellIndices;
byte[] EdgeTags = grdDat.Edges.EdgeTags;
var Cell2Edge = grdDat.Cells.Cells2Edges;
int myMpiRank = Tracker.GridDat.MpiRank;
int NoOfEdges = grdDat.Edges.Count;
int Jup = grdDat.Cells.NoOfLocalUpdatedCells;
int Jtot = grdDat.Cells.Count;
Partitioning CellPart = Tracker.GridDat.CellPartitioning;
int i0 = CellPart.i0;
var GidxExt = Tracker.GridDat.Parallel.GlobalIndicesExternalCells;
var GidxExt2Lidx = Tracker.GridDat.Parallel.Global2LocalIdx;
//double[] RefVolumes = grdDat.Grid.RefElements.Select(Kref => Kref.Volume).ToArray();
if (CellVolumes.GetLength(0) != Jtot)
throw new ArgumentException();
if (edgeArea.GetLength(0) != NoOfEdges)
throw new ArgumentException();
double EmptyEdgeTreshold = 1.0e-10; // edges with a measure blow or equal to this threshold are
// considered to be 'empty', therefore they should not be used for agglomeration;
// there is, as always, an exception: if all inner edges which belong to a
// cell that should be agglomerated, the criterion mentioned above must be ignored.
// determine agglomeration cells
// ================================
var _AccEdgesMask = new BitArray(NoOfEdges);
var AgglomCellsBitmask = new BitArray(Jup);
var _AgglomCellsEdges = new BitArray(NoOfEdges);
//var _AllowedEdges = new BitArray(NoOfEdges); _AllowedEdges.SetAll(true); // AllowedEdges.GetBitMask();
var AgglomerationPairs = new List<CellAgglomerator.AgglomerationPair>();
{
// mask for the cells in which we -- potentially -- want to do agglomeration
var AggCandidates = Tracker.Regions.GetSpeciesMask(spId).GetBitMaskWithExternal().CloneAs();
// pass 1: determine agglomeration sources
// ---------------------------------------
List<int> AgglomCellsList = new List<int>();
{
// for the present timestep
// - - - - - - - - - - - - -
CellMask suspectsForAgg = Tracker.Regions.GetCutCellMask().Intersect(Tracker.Regions.GetSpeciesMask(spId));
foreach (int jCell in suspectsForAgg.ItemEnum) {
double totVol = grdDat.Cells.GetCellVolume(jCell);
double spcVol = CellVolumes[jCell];
double alpha = AgglomerationThreshold;
spcVol = Math.Max(spcVol, 0.0);
double frac = spcVol / totVol;
//if (!(frac >= 0.0 && frac <= 1.00000001))
// throw new Exception("Strange volume fraction: cell " + jCell + ", species" + spId.ToString() + ", volume fraction =" + frac + ".");
frac = Math.Min(1.0, Math.Max(0.0, frac));
if (frac < alpha) {
// cell 'jCell' should be agglomerated to some other cell
AgglomCellsBitmask[jCell] = true;
AgglomCellsList.Add(jCell);
}
}
}
int NoTimeLev = oldTs__AgglomerationTreshold != null ? oldTs__AgglomerationTreshold.Length : 0;
if (NoTimeLev > 0) {
// for the previous timestep
// - - - - - - - - - - - - -
//ushort[][] PrevRegions = new ushort[NoTimeLev][];
//CellMask suspectsForAgg = Tracker.PreviousRegions.GetSpeciesMask(spId);
//CellMask[] suspectsForAgg = new CellMask[NoTimeLev];
//for(int itl = 0; itl < NoTimeLev; itl++) {
// PrevRegions[itl] = Tracker.RegionsHistory[-itl].LevelSetRegionsCode;
// suspectsForAgg[itl] = Tracker.RegionsHistory[-itl].GetSpeciesMask(spId);
//}
LevelSetSignCode[] signCodes = Tracker.GetLevelSetSignCodes(spId);
int NoOfLevSets = Tracker.LevelSets.Count;
for (int iTimeLev = 0; iTimeLev < NoTimeLev; iTimeLev++) {
CellMask suspectsForAgg = Tracker.RegionsHistory[-iTimeLev].GetSpeciesMask(spId);
foreach (int jCell in suspectsForAgg.ItemEnum) {
double totVol = grdDat.Cells.GetCellVolume(jCell);
double spcVol = oldCellVolumes[iTimeLev][jCell];
double alpha = oldTs__AgglomerationTreshold[iTimeLev];
spcVol = Math.Max(spcVol, 0.0);
double frac = spcVol / totVol;
frac = Math.Min(1.0, Math.Max(0.0, frac));
if (frac < alpha) {
// cell 'jCell' should be agglomerated to some other cell
if (!AgglomCellsBitmask[jCell]) {
AgglomCellsBitmask[jCell] = true;
AgglomCellsList.Add(jCell);
}
}
}
}
}
if (AgglomerateNewborn) {
for (int j = 0; j < Jup; j++) {
double vol = grdDat.Cells.GetCellVolume(j);
double volNewFrac_j = Math.Max(CellVolumes[j], 0.0) / vol;
volNewFrac_j = Math.Min(1.0, Math.Max(0.0, volNewFrac_j));
if (volNewFrac_j > NewbornAndDecasedThreshold) {
for (int nTs = 0; nTs < oldCellVolumes.Length; nTs++) {
double volOldFrac_j = Math.Max(oldCellVolumes[nTs][j], 0.0) / vol;
volOldFrac_j = Math.Min(1.0, Math.Max(0.0, volOldFrac_j));
if (volOldFrac_j <= NewbornAndDecasedThreshold) {
// cell exists at new time, but not at some old time -> newborn
int jNewbornCell = j;
AggCandidates[jNewbornCell] = false;
if (!AgglomCellsBitmask[jNewbornCell]) {
AgglomCellsList.Add(jNewbornCell);
AgglomCellsBitmask[jNewbornCell] = true;
}
}
}
}
}
}
if (AgglomerateDeceased) {
for (int j = 0; j < Jup; j++) {
double vol = grdDat.Cells.GetCellVolume(j);
double volNewFrac_j = Math.Max(CellVolumes[j], 0.0) / vol;
volNewFrac_j = Math.Min(1.0, Math.Max(0.0, volNewFrac_j));
if (volNewFrac_j <= NewbornAndDecasedThreshold) {
for (int nTs = 0; nTs < oldCellVolumes.Length; nTs++) {
double volOldFrac_j = Math.Max(oldCellVolumes[nTs][j], 0.0) / vol;
volOldFrac_j = Math.Min(1.0, Math.Max(0.0, volOldFrac_j));
if (volOldFrac_j > NewbornAndDecasedThreshold) {
// cell does not exist at new time, but at some old time -> decased
int jNewbornCell = j;
AggCandidates[jNewbornCell] = false;
if (!AgglomCellsBitmask[jNewbornCell]) {
AgglomCellsList.Add(jNewbornCell);
AgglomCellsBitmask[jNewbornCell] = true;
}
}
}
}
}
}
//*/
/*
if (AgglomerateNewborn || AgglomerateDeceased) {
//CellMask oldSpeciesCells = this.Tracker.LevelSetData.PreviousSubGrids[spId].VolumeMask;
CellMask newSpeciesCells = Tracker.Regions.GetSpeciesMask(spId);
// only accept cells with positive volume (new species cells)
BitArray newSpeciesCellsBitmask = newSpeciesCells.GetBitMask().CloneAs();
Debug.Assert(newSpeciesCellsBitmask.Count == Jup);
for (int j = 0; j < Jup; j++) {
double vol = grdDat.Cells.GetCellVolume(j);
double volFrac_j = Math.Max(CellVolumes[j], 0.0) / vol;
volFrac_j = Math.Min(1.0, Math.Max(0.0, volFrac_j));
if (volFrac_j > NewbornAndDecasedThreshold) {
Debug.Assert(newSpeciesCellsBitmask[j] == true);
} else {
newSpeciesCellsBitmask[j] = false;
}
}
newSpeciesCells = new CellMask(grdDat, newSpeciesCellsBitmask);
// only accept cells with positive volume (old species cells)
BitArray oldSpeciesCellsBitmask = new BitArray(Jup); // oldSpeciesCells.GetBitMask().CloneAs();
//Debug.Assert(oldSpeciesCellsBitmask.Count == Jup);
for (int nTs = 0; nTs < oldCellVolumes.Length; nTs++) {
MultidimensionalArray _oldCellVolumes = oldCellVolumes[nTs];
for (int j = 0; j < Jup; j++) {
double vol = grdDat.Cells.GetCellVolume(j);
double volFrac_j = Math.Max(_oldCellVolumes[j],0.0) / vol;
volFrac_j = Math.Min(1.0, Math.Max(0.0, volFrac_j));
if (volFrac_j > NewbornAndDecasedThreshold) {
//Debug.Assert(oldSpeciesCellsBitmask[j] == true);
oldSpeciesCellsBitmask[j] = true;
} else {
//oldSpeciesCellsBitmask[j] = false;
}
}
}
var oldSpeciesCells = new CellMask(grdDat, oldSpeciesCellsBitmask);
// find newborn and decased
CellMask newBorn = newSpeciesCells.Except(oldSpeciesCells);
CellMask deceased = oldSpeciesCells.Except(newSpeciesCells);
foreach (int jNewbornCell in newBorn.ItemEnum) {
AggCandidates[jNewbornCell] = false;
if (!AgglomCellsBitmask[jNewbornCell]) {
AgglomCellsList.Add(jNewbornCell);
AgglomCellsBitmask[jNewbornCell] = true;
}
//Console.WriteLine(" agglom newborn: " + jNewbornCell);
}
foreach (int jDeceasedCell in deceased.ItemEnum) {
AggCandidates[jDeceasedCell] = false;
if (!AgglomCellsBitmask[jDeceasedCell]) {
AgglomCellsList.Add(jDeceasedCell);
AgglomCellsBitmask[jDeceasedCell] = true;
}
//Console.WriteLine(" agglom deceased: " + jDeceasedCell);
}
}
//*/
// pass 2: determine agglomeration targets
// ---------------------------------------
foreach (int jCell in AgglomCellsList) {
var Cell2Edge_jCell = Cell2Edge[jCell];
// cell 'jCell' should be agglomerated to some other cell
Debug.Assert(AgglomCellsBitmask[jCell] == true);
double frac_neigh_max = -1.0;
int e_max = -1;
int jEdge_max = int.MinValue;
int jCellNeigh_max = int.MinValue;
int NoOfEdges_4_jCell = Cell2Edge_jCell.Length;
// determine if there is a non-empty edge which connects cell 'jCell'
// to some other cell
bool NonEmptyEdgeAvailable = false;
for (int e = 0; e < NoOfEdges_4_jCell; e++) { // loop over faces/neighbour cells...
int iEdge = Cell2Edge_jCell[e];
int OtherCell;
if (iEdge < 0) {
// cell 'jCell' is the OUT-cell of edge 'iEdge'
OtherCell = 0;
iEdge *= -1;
} else {
OtherCell = 1;
}
iEdge--;
int jCellNeigh = Edge2Cell[iEdge, OtherCell];
double EdgeArea_iEdge = edgeArea[iEdge];
if (jCellNeigh >= 0 && EdgeArea_iEdge > EmptyEdgeTreshold)
NonEmptyEdgeAvailable = true;
}
for (int e = 0; e < NoOfEdges_4_jCell; e++) { // loop over faces/neighbour cells...
int iEdge = Cell2Edge_jCell[e];
int OtherCell, ThisCell;
if (iEdge < 0) {
// cell 'jCell' is the OUT-cell of edge 'iEdge'
OtherCell = 0;
ThisCell = 1;
iEdge *= -1;
} else {
OtherCell = 1;
ThisCell = 0;
}
iEdge--;
double EdgeArea_iEdge = edgeArea[iEdge];
_AgglomCellsEdges[iEdge] = true;
Debug.Assert(Edge2Cell[iEdge, ThisCell] == jCell);
int jCellNeigh = Edge2Cell[iEdge, OtherCell];
if (jCellNeigh < 0 || EdgeTags[iEdge] >= GridCommons.FIRST_PERIODIC_BC_TAG || (EdgeArea_iEdge <= EmptyEdgeTreshold && NonEmptyEdgeAvailable)) {
// boundary edge, no neighbour for agglomeration
Debug.Assert(Edge2Cell[iEdge, ThisCell] == jCell, "sollte aber so sein");
continue;
}
if (!AggCandidates[jCellNeigh])
// not suitable for agglomeration
continue;
// volume fraction of neighbour cell
double spcVol_neigh = CellVolumes[jCellNeigh];
//double totVol_neigh = RefVolumes[grdDat.Cells.GetRefElementIndex(jCellNeigh)];
double totVol_neigh = grdDat.Cells.GetCellVolume(jCellNeigh);
double frac_neigh = spcVol_neigh / totVol_neigh;
// max?
if (frac_neigh > frac_neigh_max) {
frac_neigh_max = frac_neigh;
e_max = e;
jCellNeigh_max = jCellNeigh;
jEdge_max = iEdge;
}
}
if (jCellNeigh_max < 0) {
string message = ("Agglomeration failed - no candidate for agglomeration found");
if (ExceptionOnFailedAgglomeration)
throw new Exception(message);
else
Console.WriteLine(message);
} else {
_AccEdgesMask[jEdge_max] = true;
int jCellNeighRank;
if (jCellNeigh_max < Jup) {
jCellNeighRank = myMpiRank;
} else {
jCellNeighRank = CellPart.FindProcess(GidxExt[jCellNeigh_max - Jup]);
}
AgglomerationPairs.Add(new CellAgglomerator.AgglomerationPair() {
jCellTarget = jCellNeigh_max,
jCellSource = jCell,
OwnerRank4Target = jCellNeighRank,
OwnerRank4Source = myMpiRank
});
}
}
}
// store & return
// ================
return AgglomerationPairs.Select(pair => new Tuple<int, int>(pair.jCellSource, pair.jCellTarget)).ToArray();
}
}
/// <summary>
/// Like <see cref="Evaluate(double, IEnumerable{DGField}, MultidimensionalArray, double, MultidimensionalArray, BitArray, int[])"/>,
/// but with MPI-Exchange
/// </summary>
public int EvaluateParallel(double alpha, IEnumerable <DGField> Flds, MultidimensionalArray Points, double beta, MultidimensionalArray Result, BitArray UnlocatedPoints = null)
{
using (new FuncTrace()) {
MPICollectiveWatchDog.Watch();
int L = Points != null ? Points.NoOfRows : 0;
int MPIsize = m_Context.MpiSize;
int D = m_Context.SpatialDimension;
if (UnlocatedPoints != null)
{
if (UnlocatedPoints.Length != L)
{
throw new ArgumentException("Length mismatch");
}
}
// evaluate locally
// ================
var unlocated = new System.Collections.BitArray(L);
int NoOfUnlocated;
if (L > 0)
{
NoOfUnlocated = this.Evaluate(alpha, Flds, Points, beta, Result, unlocated);
}
else
{
NoOfUnlocated = 0;
}
// return, if there are no unlocalized points
// ==========================================
int TotNoOfUnlocated = NoOfUnlocated.MPISum();
if (TotNoOfUnlocated <= 0)
{
if (UnlocatedPoints != null)
{
UnlocatedPoints.SetAll(false);
}
return(0);
}
// copy unlocalized to separate array
// ==================================
double[,] localUnlocated = new double[NoOfUnlocated, D]; // MultidimensionalArray does not allow zero length -- so use double[,] instead
int[] IndexToOrgIndex = new int[NoOfUnlocated];
int u = 0;
for (int i = 0; i < L; i++)
{
if (unlocated[i])
{
localUnlocated.SetRowPt(u, Points.GetRowPt(i));
IndexToOrgIndex[u] = i;
u++;
}
}
Debug.Assert(u == NoOfUnlocated);
// collect on all ranks -- this won't scale well, but it may work
// ==============================================================
MultidimensionalArray globalUnlocated;
int[] WhoIsInterestedIn; // index: point index, corresponds with 'globalUnlocated' rows; content: rank which needs the result
int[] OriginalIndex; // index: detto; content: index which the point had on the processor that sent it.
double[][,] __globalUnlocated;
int LL;
{
__globalUnlocated = localUnlocated.MPIAllGatherO();
Debug.Assert(__globalUnlocated.Length == MPIsize);
Debug.Assert(__globalUnlocated.Select(aa => aa.GetLength(0)).Sum() == TotNoOfUnlocated);
Debug.Assert(__globalUnlocated[m_Context.MpiRank].GetLength(0) == NoOfUnlocated);
LL = TotNoOfUnlocated - NoOfUnlocated;
if (LL > 0)
{
globalUnlocated = MultidimensionalArray.Create(LL, D);
}
else
{
globalUnlocated = null;
}
WhoIsInterestedIn = new int[LL];
OriginalIndex = new int[LL];
int g = 0;
for (int r = 0; r < MPIsize; r++) // concat all point arrays from all processors
{
if (r == m_Context.MpiRank)
{
continue;
}
double[,] __globalPart = __globalUnlocated[r];
int Lr = __globalPart.GetLength(0);
if (Lr > 0)
{
globalUnlocated.ExtractSubArrayShallow(new[] { g, 0 }, new[] { g + Lr - 1, D - 1 }).Acc2DArray(1.0, __globalPart);
}
for (int i = 0; i < Lr; i++)
{
WhoIsInterestedIn[i + g] = r;
OriginalIndex[i + g] = i;
}
g += Lr;
}
}
// try to evaluate the so-far-unlocalized points
// ---------------------------------------------
var unlocated2 = new System.Collections.BitArray(LL);
var Result2 = LL > 0 ? MultidimensionalArray.Create(LL, Flds.Count()) : null;
int NoOfUnlocated2 = LL > 0 ? this.Evaluate(1.0, Flds, globalUnlocated, 0.0, Result2, unlocated2) : 0;
// backward MPI sending
// --------------------
IDictionary <int, EvaluateParallelHelper> resultFromOtherProcs;
{
var backSend = new Dictionary <int, EvaluateParallelHelper>();
for (int ll = 0; ll < LL; ll++)
{
if (!unlocated2[ll])
{
int iTarget = WhoIsInterestedIn[ll];
Debug.Assert(iTarget != m_Context.MpiRank);
if (!backSend.TryGetValue(iTarget, out EvaluateParallelHelper eph))
{
eph = new EvaluateParallelHelper();
backSend.Add(iTarget, eph);
}
eph.OriginalIndices.Add(OriginalIndex[ll]);
eph.Results.Add(Result2.GetRow(ll));
}
}
resultFromOtherProcs = SerialisationMessenger.ExchangeData(backSend);
}
// fill the results from other processors
// ======================================
foreach (var res in resultFromOtherProcs.Values)
{
int K = res.OriginalIndices.Count();
Debug.Assert(res.OriginalIndices.Count == res.Results.Count);
for (int k = 0; k < K; k++)
{
int iOrg = IndexToOrgIndex[res.OriginalIndices[k]];
if (unlocated[iOrg] == true)
{
NoOfUnlocated--;
}
unlocated[iOrg] = false;
Result.AccRow(iOrg, alpha, res.Results[k]);
}
}
// Return
// ======
if (UnlocatedPoints != null)
{
for (int l = 0; l < L; l++)
{
UnlocatedPoints[l] = unlocated[l];
}
}
return(NoOfUnlocated);
}
}
/// <summary>
/// ctor
/// </summary>
/// <param name="volMask">
/// volume mask for those cells which should be contained in the subgrid
/// </param>
public SubGrid(CellMask volMask)
{
MPICollectiveWatchDog.Watch();
this.m_VolumeMask = volMask;
m_GridData = volMask.GridData;
}
/// <summary>
/// constructs a 1D-grid consisting of multiple (not connected) segments
/// </summary>
/// <param name="nodeSets">
/// a collection of node segments;
/// if running with more than one MPI process, only the input on rank 0 is considered
/// </param>
/// <param name="periodic"></param>
static public Grid1D LineGrid(ICollection <double[]> nodeSets, bool periodic = false)
{
using (new FuncTrace()) {
MPICollectiveWatchDog.Watch();
int size, rank;
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out size);
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out rank);
// Don't forget minus one (sets contain _nodes_, not cells)!
int globalNumberOfCells = nodeSets.Select(s => s.Length - 1).Sum();
int globalIndexFirstLocalCell = globalNumberOfCells * rank / size;
int globalIndexFirstExternalCell = globalNumberOfCells * (rank + 1) / size;
Grid1D grid = new Grid1D();
if (Math.Abs(globalIndexFirstLocalCell - globalIndexFirstExternalCell) <= 0)
{
// No cells on current rank (too few cells?)
return(grid);
}
int localNoOfCells = globalIndexFirstExternalCell - globalIndexFirstLocalCell;
grid.Cells = new Cell[localNoOfCells];
byte periodicTag = 0;
if (periodic)
{
double[][] left = { new double[] { nodeSets.First().First() } };
double[][] right = { new double[] { nodeSets.Last().Last() } };
grid.ConstructPeriodicEdgeTrafo(
right, new double[] { 1.0 }, left, new double[] { 1.0 }, out periodicTag);
grid.EdgeTagNames.Add(periodicTag, "Periodic-X");
}
int globalCellIndex = -1;
int localCellIndex = 0;
foreach (double[] nodesInSegment in nodeSets)
{
int noOfCellsInSegment = nodesInSegment.Length - 1;
for (int i = 0; i < noOfCellsInSegment; i++)
{
globalCellIndex++;
// Handled on other processor
if (globalCellIndex < globalIndexFirstLocalCell)
{
continue;
}
else if (globalCellIndex >= globalIndexFirstExternalCell)
{
break;
}
if (nodesInSegment[i] >= nodesInSegment[i + 1])
{
throw new ArgumentException(
"Nodes must be provided in strictly ascending order");
}
Cell cell = new Cell();
cell.GlobalID = globalCellIndex;
cell.Type = CellType.Line_2;
cell.TransformationParams = MultidimensionalArray.Create(2, 1);
cell.TransformationParams[0, 0] = nodesInSegment[i];
cell.TransformationParams[1, 0] = nodesInSegment[i + 1];
cell.NodeIndices = new int[] {
globalCellIndex,
globalCellIndex + 1
};
if (periodic)
{
if (globalCellIndex == 0)
{
(new CellFaceTag()
{
EdgeTag = periodicTag,
PeriodicInverse = true,
ConformalNeighborship = true,
FaceIndex = (int)Line.Edge.Left,
NeighCell_GlobalID = globalNumberOfCells - 1
}).AddToArray(ref cell.CellFaceTags);
}
else if (globalCellIndex == globalNumberOfCells - 1)
{
(new CellFaceTag()
{
EdgeTag = periodicTag,
PeriodicInverse = false,
ConformalNeighborship = true,
FaceIndex = (int)Line.Edge.Right,
NeighCell_GlobalID = 0
}).AddToArray(ref cell.CellFaceTags);
}
}
grid.Cells[localCellIndex] = cell;
localCellIndex++;
}
}
return(grid);
}
}
/// <summary>
/// Solves a linear regression cost model
/// </summary>
/// <returns></returns>
public int[] GetEstimatedCellCosts()
{
MPICollectiveWatchDog.Watch();
int noOfEstimates = localCellCounts.Count;
Debug.Assert(noOfEstimates == localRunTimeEstimates.Count);
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out int MpiSize);
MultidimensionalArray SendBuf = MultidimensionalArray.Create(
noOfEstimates, CurrentPerformanceClassCount + 1);
for (int i = 0; i < noOfEstimates; i++)
{
for (int j = 0; j < CurrentPerformanceClassCount; j++)
{
SendBuf[i, j] = localCellCounts[i][j];
}
SendBuf[i, CurrentPerformanceClassCount] = localRunTimeEstimates[i];
}
MultidimensionalArray RecvBuf = MultidimensionalArray.Create(
MpiSize, noOfEstimates, CurrentPerformanceClassCount + 1);
unsafe
{
fixed(double *pSendBuf = &SendBuf.Storage[0], pRecvBuf = &RecvBuf.Storage[0])
{
csMPI.Raw.Allgather(
(IntPtr)pSendBuf, SendBuf.Length, csMPI.Raw._DATATYPE.DOUBLE,
(IntPtr)pRecvBuf, SendBuf.Length, csMPI.Raw._DATATYPE.DOUBLE,
csMPI.Raw._COMM.WORLD);
}
}
#if DEBUG
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out int rank);
for (int i = 0; i < noOfEstimates; i++)
{
for (int j = 0; j < CurrentPerformanceClassCount; j++)
{
Debug.Assert(RecvBuf[rank, i, j] == SendBuf[i, j]);
Debug.Assert(RecvBuf[rank, i, j] == localCellCounts[i][j]);
}
Debug.Assert(RecvBuf[rank, i, CurrentPerformanceClassCount] == SendBuf[i, CurrentPerformanceClassCount]);
Debug.Assert(RecvBuf[rank, i, CurrentPerformanceClassCount] == localRunTimeEstimates[i]);
}
#endif
// TO DO: MAKE EFFICIENT
int noOfRows = MpiSize * noOfEstimates;
int noOfColumns = CurrentPerformanceClassCount;
MultidimensionalArray matrix = MultidimensionalArray.Create(noOfRows, noOfColumns);
MultidimensionalArray rhs = MultidimensionalArray.Create(noOfRows, 1);
for (int i = 0; i < MpiSize; i++)
{
for (int j = 0; j < noOfEstimates; j++)
{
for (int k = 0; k < CurrentPerformanceClassCount; k++)
{
matrix[i * noOfEstimates + j, k] = RecvBuf[i, j, k];
}
rhs[i * noOfEstimates + j, 0] = RecvBuf[i, j, CurrentPerformanceClassCount];
}
}
//// REMOVE IDENTICAL ROWS
//List<int> obsoleteRows = new List<int>();
//List<double> averages = new List<double>();
//for (int row0 = 0; row0 < matrix.NoOfRows; row0++) {
// if (obsoleteRows.Contains(row0)) {
// continue;
// }
// List<int> identicalRows = new List<int>();
// for (int row1 = row0 + 1; row1 < matrix.NoOfRows; row1++) {
// bool rowIsIdentical = true;
// for (int col = 0; col < matrix.NoOfCols; col++) {
// rowIsIdentical &= (matrix[row0, col] == matrix[row1, col]);
// }
// if (rowIsIdentical) {
// identicalRows.Add(row1);
// }
// }
// obsoleteRows.AddRange(identicalRows);
// double average = (rhs[row0, 0] + identicalRows.Sum(i => rhs[i, 0])) / (1 + identicalRows.Count);
// averages.Add(average);
//}
//MultidimensionalArray reducedMatrix = MultidimensionalArray.Create(
// matrix.NoOfRows - obsoleteRows.Count(), matrix.NoOfCols);
//MultidimensionalArray reducedRHS = MultidimensionalArray.Create(
// reducedMatrix.NoOfRows, 1);
//int reducedRow = 0;
//foreach (int row in Enumerable.Range(0, matrix.NoOfRows).Except(obsoleteRows)) {
// for (int col = 0; col < reducedMatrix.NoOfCols; col++) {
// reducedMatrix[reducedRow, col] = matrix[row, col];
// }
// reducedRHS[reducedRow, 0] = averages[reducedRow];
// reducedRow++;
//}
if (MpiSize > 1 && rhs.Length > MpiSize)
{
MultidimensionalArray costs = MultidimensionalArray.Create(CurrentPerformanceClassCount, 1);
matrix.LeastSquareSolve(costs, rhs);
//reducedMatrix.LeastSquareSolve(costs, reducedRHS);
double minCost = costs.Min();
double maxCost = costs.Max();
int[] intCost = costs.Storage.Select(
cost => Math.Max(1, (int)Math.Round(1e3 * (cost / maxCost)))).ToArray();
for (int i = 0; i < CurrentPerformanceClassCount; i++)
{
Console.WriteLine(
"Cost of cell type {0}: \tabs:{1:0.##E-00} \trel1: {2:0.0%}\trel2: {3:0.0%}\tint: {4}",
i,
costs[i, 0],
costs[i, 0] / minCost,
costs[i, 0] / maxCost,
intCost[i]);
}
int[] cellCosts = new int[currentCellToPerformanceClassMap.Length];
for (int j = 0; j < cellCosts.Length; j++)
{
cellCosts[j] = intCost[currentCellToPerformanceClassMap[j]];
}
return(cellCosts);
}
else
{
return(null);
}
}