/// <summary>
/// gets a subgrid of width <paramref name="FieldWidth"/>, around level set No. <paramref name="levSetIdx"/>;
/// </summary>
/// <remarks>
/// In contrast to <see cref="SubGrid"/>'s, <see cref="CellMask"/>'s are not MPI-collective, and should therefore be preferred.
/// </remarks>
public SubGrid GetNearFieldSubgrid4LevSet(int levSetIdx, int FieldWidth)
{
MPICollectiveWatchDog.Watch();
if (FieldWidth > m_owner.m_NearRegionWidth)
{
throw new ArgumentException("Near-" + FieldWidth + " cannot be acquired, because this tracker is set to detect at most Near-" + m_owner.m_NearRegionWidth + ".", "FieldWidth");
}
if (levSetIdx < 0 || levSetIdx >= this.m_owner.NoOfLevelSets)
{
throw new IndexOutOfRangeException();
}
if (m_NearField4LevelSet == null || m_NearField4LevelSet.GetLength(1) != this.m_owner.m_NearRegionWidth)
{
m_NearField4LevelSet = new SubGrid[this.m_owner.NoOfLevelSets, this.m_owner.m_NearRegionWidth + 1];
}
if (m_NearField4LevelSet[levSetIdx, FieldWidth] == null)
{
// create subgrid
m_NearField4LevelSet[levSetIdx, FieldWidth] = new SubGrid(GetNearMask4LevSet(levSetIdx, FieldWidth));
}
return(m_NearField4LevelSet[levSetIdx, FieldWidth]);
}
/// <summary>
/// ctor
/// </summary>
public PARDISOSolver()
{
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out rank);
//if (rank == 0)
// Debugger.Launch();
}
/// <summary>
/// accumulates the derivative of DG field <paramref name="f"/>
/// (along the <paramref name="d"/>-th axis) times <paramref name="alpha"/>
/// to this field, i.e. <br/>
/// this = this + <paramref name="alpha"/>* \f$ \frac{\partial}{\partial x_d} \f$ <paramref name="f"/>;
/// </summary>
/// <param name="f"></param>
/// <param name="d">
/// 0 for the x-derivative, 1 for the y-derivative, 2 for the z-derivative
/// </param>
/// <param name="alpha">
/// scaling of <paramref name="f"/>;
/// </param>
/// <param name="em">
/// An optional restriction to the domain in which the derivative is computed (it may, e.g.
/// be only required in boundary cells, so a computation over the whole domain
/// would be a waste of computation power. A proper execution mask for this case would be e.g.
/// <see cref="BoSSS.Foundation.Grid.GridData.BoundaryCells"/>.)<br/>
/// if null, the computation is carried out in the whole domain
/// </param>
override public void Derivative(double alpha, DGField f, int d, CellMask em)
{
using (var tr = new FuncTrace()) {
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
if (this.Basis.Degree < f.Basis.Degree - 1)
{
throw new ArgumentException("cannot compute derivative because of incompatible basis functions.", "f");
}
if (f.Basis.GetType() != this.Basis.GetType())
{
throw new ArgumentException("cannot compute derivative because of incompatible basis functions.", "f");
}
int D = GridDat.SpatialDimension;
if (d < 0 || d >= D)
{
throw new ArgumentException("spatial dimension out of range.", "d");
}
Quadrature.EdgeQuadratureScheme _qInsEdge;
Quadrature.CellQuadratureScheme _qInsVol;
{
_qInsEdge = (new Quadrature.EdgeQuadratureScheme(false, EdgeMask.GetEmptyMask(this.GridDat)));
_qInsVol = (new Quadrature.CellQuadratureScheme(true, em));
}
var op = (new BrokenDerivativeForm(d)).Operator(1, g => _qInsEdge, g => _qInsVol);
op.Evaluate(alpha, 1.0, f.Mapping, null, this.Mapping);
}
}
/// <summary>
/// gets a subgrid of width <paramref name="FieldWidth"/>, around any level set
/// </summary>
/// <remarks>
/// In contrast to <see cref="SubGrid"/>'s, <see cref="CellMask"/>'s are not MPI-collective, and should therefore be preferred.
/// </remarks>
public SubGrid GetNearFieldSubgrid(int FieldWidth)
{
MPICollectiveWatchDog.Watch();
if (FieldWidth > m_owner.m_NearRegionWidth)
{
throw new ArgumentException("Near-" + FieldWidth + " cannot be acquired, because this tracker is set to detect at most Near-" + m_owner.m_NearRegionWidth + ".", "FieldWidth");
}
if (m_owner.NoOfLevelSets == 1)
{
// if there is only one Level Set, no need to separate between
// cells-cut-by-any-level-set and cells-cut-by-a-specific-level-set
return(GetNearFieldSubgrid4LevSet(0, FieldWidth));
}
if (m_NearField == null)
{
m_NearField = new SubGrid[m_owner.m_NearRegionWidth + 1];
}
if (m_NearField[FieldWidth] == null)
{
m_NearField[FieldWidth] = new SubGrid(GetNearFieldMask(FieldWidth));
}
return(m_NearField[FieldWidth]);
}
/// <summary>
/// Determines the admissible step-size within
/// <paramref name="subGrid"/> by calling
/// <see cref="GetLocalStepSize"/> for all contiguous chunks of cells.
/// </summary>
/// <param name="subGrid">
/// The sub-grid for which the step size shall be determined. If null
/// is given, the full domain will be considered.
/// </param>
/// <returns>
/// The maximum step size dictated by the CFL restriction in the given
/// <paramref name="subGrid"/>.
/// </returns>
/// <remarks>
/// This is a collective call which requires all processes to
/// synchronize.
/// </remarks>
public double GetGloballyAdmissibleStepSize(SubGrid subGrid = null)
{
MPICollectiveWatchDog.Watch();
subGrid = subGrid ?? new SubGrid(CellMask.GetFullMask(gridData));
double maxTimeStep = double.MaxValue;
double globalMaxTimeStep;
System.Exception e = null;
try {
foreach (Chunk chunk in subGrid.VolumeMask)
{
maxTimeStep = Math.Min(
maxTimeStep,
GetLocalStepSize(chunk.i0, chunk.Len));
}
} catch (System.Exception ee) {
e = ee;
}
e.ExceptionBcast();
unsafe
{
csMPI.Raw.Allreduce(
(IntPtr)(&maxTimeStep),
(IntPtr)(&globalMaxTimeStep),
1,
csMPI.Raw._DATATYPE.DOUBLE,
csMPI.Raw._OP.MIN,
csMPI.Raw._COMM.WORLD);
}
return(dtFraction * globalMaxTimeStep);
}
/// <summary>
/// accumulates the derivative of DG field <paramref name="f"/>
/// (along the <paramref name="d"/>-th axis) times <paramref name="alpha"/>
/// to this field, i.e. <br/>
/// this = this + <paramref name="alpha"/>* \f$ \frac{\partial}{\partial x_d} \f$ <paramref name="f"/>;
/// </summary>
/// <param name="f"></param>
/// <param name="d">
/// 0 for the x-derivative, 1 for the y-derivative, 2 for the
/// z-derivative
/// </param>
/// <param name="alpha">
/// scaling of <paramref name="f"/>;
/// </param>
/// <param name="optionalSubGrid">
/// An optional restriction to the domain in which the derivative is
/// computed (it may, e.g. be only required in boundary cells, so a
/// computation over the whole domain would be a waste of computational
/// power. A proper execution mask would be see e.g.
/// <see cref="GridData.BoundaryCells"/>.)
/// <br/>
/// if null, the computation is carried out in the whole domain.
/// </param>
/// <param name="bndMode">
/// if a sub-grid is provided, this determines how the sub-grid
/// boundary should be treated.
/// </param>
/// <remarks>
/// The derivative is calculated using Riemann flux functions
/// (central difference);<br/>
/// In comparison to
/// <see cref="Derivative(double, DGField, int, CellMask)"/>, this method
/// should be slower, but produce more sane results, especially for
/// fields of low polynomial degree (0 or 1);
/// </remarks>
virtual public void DerivativeByFlux(double alpha, DGField f, int d, SubGrid optionalSubGrid = null, SubGridBoundaryModes bndMode = SubGridBoundaryModes.OpenBoundary)
{
int D = this.Basis.GridDat.SpatialDimension;
if (d < 0 || d >= D)
{
throw new ArgumentException("spatial dimension out of range.", "d");
}
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
EdgeMask emEdge = (optionalSubGrid != null) ? optionalSubGrid.AllEdgesMask : null;
CellMask emVol = (optionalSubGrid != null) ? optionalSubGrid.VolumeMask : null;
SpatialOperator d_dx = new SpatialOperator(1, 1, QuadOrderFunc.Linear(), "in", "out");
d_dx.EdgeQuadraturSchemeProvider = g => new Quadrature.EdgeQuadratureScheme(true, emEdge);
d_dx.VolumeQuadraturSchemeProvider = g => new Quadrature.CellQuadratureScheme(true, emVol);
var flux = CreateDerivativeFlux(d, f.Identification);
d_dx.EquationComponents["out"].Add(flux);
d_dx.Commit();
var ev = d_dx.GetEvaluatorEx(
new CoordinateMapping(f), null, this.Mapping);
if (optionalSubGrid != null)
{
ev.ActivateSubgridBoundary(optionalSubGrid.VolumeMask, bndMode);
}
ev.Evaluate <CoordinateVector>(alpha, 1.0, this.CoordinateVector);
}
/// <summary>
/// integrates this field over the domain specified in <paramref name="volumemask"/>
/// </summary>
/// <param name="volumemask">
/// an optional volume mask; if null, the whole grid is taken;
/// </param>
public double IntegralOver(CellMask volumemask)
{
using (new FuncTrace()) {
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
if (volumemask == null)
{
volumemask = CellMask.GetFullMask(Basis.GridDat);
}
double acc = 0;
foreach (var chunk in volumemask)
{
int iE = chunk.i0 + chunk.Len;
for (int j = chunk.i0; j < iE; j++)
{
double mv = GetMeanValue(j);
double vol = Basis.GridDat.iLogicalCells.GetCellVolume(j);
acc += mv * vol;
}
}
double accglob = double.NaN;
unsafe {
csMPI.Raw.Allreduce((IntPtr)(&acc), (IntPtr)(&accglob), 1, csMPI.Raw._DATATYPE.DOUBLE, csMPI.Raw._OP.SUM, csMPI.Raw._COMM.WORLD);
}
return(accglob);
}
}
/// <summary>
/// ctor
/// </summary>
/// <param name="_MPI_comm">the MPI communicator for this messenger (see <see cref="MPI_comm"/>);</param>
public SerialisationMessenger(MPI_Comm _MPI_comm)
{
MPICollectiveWatchDog.Watch(_MPI_comm);
m_MPI_comm = _MPI_comm;
csMPI.Raw.Comm_Rank(_MPI_comm, out m_Rank);
csMPI.Raw.Comm_Size(_MPI_comm, out m_Size);
m_MyCommPaths = new byte[m_Size];
m_AllCommPaths = new byte[m_Size, m_Size];
m_ReceiveObjectSizes = new int[m_Size];
m_SendObjectSizes = new int[m_Size];
m_Requests = new MPI_Request[m_Size * 4];
m_TransmittCalled = new BitArray(m_Size, false);
unsafe {
int myTag = TagCnt;
TagCnt += 13;
if (m_Rank == 0)
{
m_MyTagOffset = myTag;
}
csMPI.Raw.Bcast((IntPtr)(&myTag), 1, csMPI.Raw._DATATYPE.INT, 0, m_MPI_comm);
if (m_Rank > 0)
{
m_MyTagOffset = myTag;
}
}
}
void CheckIfSuitableForSaving(IGrid grid)
{
if (grid.ID.Equals(Guid.Empty))
{
throw new ApplicationException("cannot save grid with empty Guid (Grid Guid is " + Guid.Empty.ToString() + ");");
}
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
}
/// <summary>
/// returns the number of non-zero elements in the matrix
/// (on the current MPI process);
/// </summary>
/// <returns></returns>
static public int GetTotalNoOfNonZeros(this IMutableMatrixEx M)
{
MPICollectiveWatchDog.Watch(M.RowPartitioning.MPI_Comm);
int odnz = M.GetTotalNoOfNonZerosPerProcess();
int odnz_global = odnz.MPISum(M.MPI_Comm);
return(odnz_global);
}
/// <summary>
/// converts <paramref name="M"/> into suitable structures for PARDISO;
/// </summary>
/// <param name="M"></param>
public void DefineMatrix(IMutableMatrixEx M)
{
if (m_OrgMatrix != null)
{
throw new ApplicationException("matrix is already defined. 'DefineMatrix'-method can be invoked only once in the lifetime of this object.");
}
m_MpiRank = M.RowPartitioning.MpiRank;
m_OrgMatrix = M;
MPICollectiveWatchDog.Watch(this.MpiComm);
}
/// <summary>
/// ctor
/// </summary>
/// <param name="volMask">
/// volume mask for those cells which should be contained in the subgrid
/// </param>
public SubGrid(CellMask volMask)
{
MPICollectiveWatchDog.Watch();
if (volMask.MaskType != MaskType.Logical)
{
throw new ArgumentException();
}
this.m_VolumeMask = volMask;
m_GridData = volMask.GridData;
}
/// <summary>
/// L-infinity - norm of this field;<br/>
/// This is a collective call, it must be invoked by all
/// MPI processes within the communicator; internally, it invokes MPI_Allreduce;
/// </summary>
/// <returns>
/// on all invoking MPI processes, the L-inv norm of
/// this field (over all processors);
/// </returns>
/// <remarks>
/// to find the maximum value, this field is evaluated on each vertex and the center of the simplex.
/// </remarks>
public double LinfNorm(CellMask cm = null)
{
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
using (new FuncTrace()) {
double Min, Max;
int j, k;
GetExtremalValues(out Min, out Max, out j, out k, cm);
return(Math.Max(Math.Abs(Max), Math.Abs(Min)));
}
}
/// <summary>
/// computes the L2 - norm based on Parceval's equality, for a specific
/// polynomial degree <paramref name="deg"/>.
/// </summary>
/// <remarks>
/// exploiting Parcevals equality for orthonormal systems, this
/// function is (should be?) much faster than
/// <see cref="DGField.L2Norm(CellMask)"/>
/// </remarks>
public virtual double L2NormPerMode(int deg, CellMask _cm = null)
{
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
using (new FuncTrace()) {
var DGCoordinates = this.Coordinates;
IEnumerable <Chunk> cm = _cm;
if (cm == null)
{
cm = new Chunk[] { new Chunk()
{
i0 = 0, Len = GridDat.iLogicalCells.NoOfLocalUpdatedCells
} }
}
;
if (this.Basis.IsOrthonormal == false)
{
throw new NotSupportedException("Not supported for non-orthonormal basis.");
}
var polys = this.Basis.Polynomials;
double nrm2_2 = 0;
int N = this.m_Basis.MaximalLength;
foreach (Chunk c in cm)
{
int JE = c.Len + c.i0;
for (int j = c.i0; j < JE; j++)
{
int iKref = this.GridDat.iGeomCells.GetRefElementIndex(j);
int N0 = polys[iKref].FirstPolynomialIndexforDegree(deg);
int NE = N0 + polys[iKref].NoOfPolynomialsPerDegree(deg);
for (int n = N0; n < NE; n++)
{
double k = DGCoordinates[j, n];
nrm2_2 += k * k;
}
}
}
double nrmtot = 0;
unsafe
{
csMPI.Raw.Allreduce((IntPtr)(&nrm2_2), (IntPtr)(&nrmtot), 1, csMPI.Raw._DATATYPE.DOUBLE, csMPI.Raw._OP.SUM, csMPI.Raw._COMM.WORLD);
}
return(Math.Sqrt(nrmtot));
}
}
/// <summary>
/// This call computes an integral measure which may depend on
/// this <see cref="DGField"/> an the given <paramref name="function"/>;
/// This is a collective call, it must be invoked by all
/// MPI processes within the communicator; internally, it invokes MPI_Allreduce;
/// </summary>
/// <param name="function"></param>
/// <param name="rule">
/// composite quadrature rule.
/// </param>
/// <param name="Map">
/// Arbiter mapping applied to the values of this field and
/// <paramref name="function"/> at some point, which is finally integrated.
/// E.g., the mapping for an L2-error would be \f$ (\vec{x},a,b) => (a - b)^2 \f$,
/// where \f$ a \f$ is the value of this field at some point \f$ \vec{x} \f$ and
/// \f$ b \f$ is the value of <paramref name="function"/> at \f$ \vec{x} \f$.
/// </param>
/// <returns>
/// on all invoking MPI processes, the L2 norm of
/// this field minus <paramref name="function"/>
/// </returns>
public double LxError(ScalarFunctionEx function, Func <double[], double, double, double> Map, ICompositeQuadRule <QuadRule> rule)
{
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
using (new FuncTrace()) {
LxNormQuadrature l2nq = new LxNormQuadrature(this, function, Map, rule);
l2nq.Execute();
double nrmtot = l2nq.LxNorm.MPISum();
return(nrmtot);
}
}
/// <summary>
/// Estimates the cost imbalance between all processes using the given
/// <paramref name="estimator"/>
/// </summary>
/// <param name="estimator"></param>
/// <returns>
/// A number between 0 and 1 which represents an estimate of the load
/// imbalance in percent
/// </returns>
public static double ImbalanceEstimate(this ICellCostEstimator estimator)
{
MPICollectiveWatchDog.Watch();
double localCost = estimator.EstimatedLocalCost;
double[] allCosts = localCost.MPIAllGather();
double minCost = allCosts.Min();
double maxCost = allCosts.Max();
double imbalance = (maxCost - minCost) / Math.Max(double.Epsilon, maxCost);
return(imbalance);
}
/// <summary>
/// computes the L2 - norm based on Parceval's equality
/// </summary>
/// <remarks>
/// exploiting Parcevals equality for orthonormal systems, this
/// function is (should be?) much faster than
/// <see cref="DGField.L2Norm(CellMask)"/>
/// </remarks>
public override double L2Norm(CellMask cm)
{
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
using (new FuncTrace()) {
var DGCoordinates = this.Coordinates;
double nrm2_2 = 0;
if (cm == null)
{
int J = GridDat.iLogicalCells.NoOfLocalUpdatedCells;
int N = this.m_Basis.MaximalLength;
for (int j = 0; j < J; j++)
{
for (int n = 0; n < N; n++)
{
double k = DGCoordinates[j, n];
nrm2_2 += k * k;
}
}
}
else
{
int N = this.m_Basis.MaximalLength;
foreach (var c in cm)
{
int JE = c.Len + c.i0;
for (int j = c.i0; j < JE; j++)
{
for (int n = 0; n < N; n++)
{
double k = DGCoordinates[j, n];
nrm2_2 += k * k;
}
}
}
}
double nrmtot = 0;
unsafe
{
csMPI.Raw.Allreduce((IntPtr)(&nrm2_2), (IntPtr)(&nrmtot), 1, csMPI.Raw._DATATYPE.DOUBLE, csMPI.Raw._OP.SUM, csMPI.Raw._COMM.WORLD);
}
return(Math.Sqrt(nrmtot));
}
}
/// <summary>
/// Equivalent to <see cref="GetSpeciesSubGrid(string)"/>
/// </summary>
public SubGrid GetSpeciesSubGrid(SpeciesId specId)
{
MPICollectiveWatchDog.Watch();
if (m_SpeciesSubGrids == null)
{
m_SpeciesSubGrids = new Dictionary <SpeciesId, SubGrid>();
}
if (!m_SpeciesSubGrids.ContainsKey(specId))
{
CellMask cm = GetSpeciesMask(specId);
m_SpeciesSubGrids.Add(specId, new SubGrid(cm));
}
return(this.m_SpeciesSubGrids[specId]);
}
/// <summary>
/// Computes <see cref="SubgridIndex2LocalCellIndex"/>
/// </summary>
/// <returns></returns>
private int[] ComputeSubgridIndex2LocalCellIndex()
{
MPICollectiveWatchDog.Watch();
CellMask msk = m_VolumeMask;
int[] subgridIndex2LocalCellIndex = new int[msk.NoOfItemsLocally_WithExternal];
int jj = 0;
foreach (Chunk c in msk.GetEnumerableWithExternal())
{
for (int k = 0; k < c.Len; k++)
{
subgridIndex2LocalCellIndex[jj] = c.i0 + k;
jj++;
}
}
return(subgridIndex2LocalCellIndex);
}
/// <summary>
/// returns a bitmask that contains also information about external/ghost cells.
/// </summary>
public BitArray GetBitMaskWithExternal()
{
if (base.MaskType != MaskType.Logical)
{
throw new NotSupportedException();
}
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
int JE = this.GridData.iLogicalCells.Count;
int J = this.GridData.iLogicalCells.NoOfLocalUpdatedCells;
int MpiSize = this.GridData.CellPartitioning.MpiSize;
//Debugger.Break();
if (MpiSize > 1)
{
// more than one MPI process
// +++++++++++++++++++++++++
if (m_BitMaskWithExternal == null)
{
m_BitMaskWithExternal = new BitArray(JE, false);
// inner cells
foreach (Chunk c in this)
{
for (int i = 0; i < c.Len; i++)
{
m_BitMaskWithExternal[i + c.i0] = true;
}
}
m_BitMaskWithExternal.MPIExchange(this.GridData);
}
return(m_BitMaskWithExternal);
}
else
{
// single - processor mode
// +++++++++++++++++++++++
return(base.GetBitMask());
}
}
/// <summary>
/// computes the inner product of fields <paramref name="a"/> and <paramref name="b"/>
/// </summary>
static public double InnerProduct(DGField a, DGField b, CellQuadratureScheme quadScheme = null)
{
using (new FuncTrace()) {
if (!Object.ReferenceEquals(a.GridDat, b.GridDat))
{
throw new ApplicationException("fields must be defined on the same grid.");
}
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
InnerProductQuadrature ipq = new InnerProductQuadrature(a, b, quadScheme, a.Basis.Degree + b.Basis.Degree);
ipq.Execute();
unsafe {
double innerProdTot = double.NaN;
double InnerProdLoc = ipq.m_InnerProd;
csMPI.Raw.Allreduce((IntPtr)(&InnerProdLoc), (IntPtr)(&innerProdTot), 1, csMPI.Raw._DATATYPE.DOUBLE, csMPI.Raw._OP.SUM, csMPI.Raw._COMM.WORLD);
return(innerProdTot);
}
}
}
/// <summary>
/// returns the subgrid containing all cells in which the sign of the level set function #<paramref name="LevelSetIndex"/>
/// is at least in one point equal to <paramref name="sign"/>.
/// </summary>
public SubGrid GetLevelSetWing(int LevelSetIndex, double sign)
{
MPICollectiveWatchDog.Watch();
if (sign == 0.0)
{
throw new ArgumentException("must be either positive or negative");
}
if (LevelSetIndex < 0 || LevelSetIndex >= m_owner.m_LevelSetHistories.Count)
{
throw new IndexOutOfRangeException("invalid level set index");
}
int _sign = Math.Sign(sign);
int iwing = _sign * (LevelSetIndex + 1);
if (m_LevelSetWings == null)
{
m_LevelSetWings = new SortedDictionary <int, SubGrid>();
}
if (!m_LevelSetWings.ContainsKey(iwing))
{
int J = m_owner.GridDat.Cells.NoOfLocalUpdatedCells;
BitArray mask = new BitArray(J);
for (int j = 0; j < J; j++)
{
int dist = DecodeLevelSetDist(m_LevSetRegions[j], LevelSetIndex);
mask[j] = (dist * _sign >= 0);
}
SubGrid LevelSetWing = new SubGrid(new CellMask(m_owner.GridDat, mask));
m_LevelSetWings.Add(iwing, LevelSetWing);
}
return(m_LevelSetWings[iwing]);
}
/// <summary>
/// constructs a 1D-grid consisting of multiple (not connected) segments
/// </summary>
/// <param name="nodeSets">
/// a collection of node segments;
/// if running with more than one MPI process, only the input on rank 0 is considered
/// </param>
/// <param name="periodic"></param>
static public Grid1D LineGrid(ICollection <double[]> nodeSets, bool periodic = false)
{
using (new FuncTrace()) {
MPICollectiveWatchDog.Watch();
int size, rank;
csMPI.Raw.Comm_Size(csMPI.Raw._COMM.WORLD, out size);
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out rank);
// Don't forget minus one (sets contain _nodes_, not cells)!
int globalNumberOfCells = nodeSets.Select(s => s.Length - 1).Sum();
int globalIndexFirstLocalCell = globalNumberOfCells * rank / size;
int globalIndexFirstExternalCell = globalNumberOfCells * (rank + 1) / size;
Grid1D grid = new Grid1D();
if (Math.Abs(globalIndexFirstLocalCell - globalIndexFirstExternalCell) <= 0)
{
// No cells on current rank (too few cells?)
return(grid);
}
int localNoOfCells = globalIndexFirstExternalCell - globalIndexFirstLocalCell;
grid.Cells = new Cell[localNoOfCells];
byte periodicTag = 0;
if (periodic)
{
double[][] left = { new double[] { nodeSets.First().First() } };
double[][] right = { new double[] { nodeSets.Last().Last() } };
grid.ConstructPeriodicEdgeTrafo(
right, new double[] { 1.0 }, left, new double[] { 1.0 }, out periodicTag);
grid.EdgeTagNames.Add(periodicTag, "Periodic-X");
}
int globalCellIndex = -1;
int localCellIndex = 0;
foreach (double[] nodesInSegment in nodeSets)
{
int noOfCellsInSegment = nodesInSegment.Length - 1;
for (int i = 0; i < noOfCellsInSegment; i++)
{
globalCellIndex++;
// Handled on other processor
if (globalCellIndex < globalIndexFirstLocalCell)
{
continue;
}
else if (globalCellIndex >= globalIndexFirstExternalCell)
{
break;
}
if (nodesInSegment[i] >= nodesInSegment[i + 1])
{
throw new ArgumentException(
"Nodes must be provided in strictly ascending order");
}
Cell cell = new Cell();
cell.GlobalID = globalCellIndex;
cell.Type = CellType.Line_2;
cell.TransformationParams = MultidimensionalArray.Create(2, 1);
cell.TransformationParams[0, 0] = nodesInSegment[i];
cell.TransformationParams[1, 0] = nodesInSegment[i + 1];
cell.NodeIndices = new int[] {
globalCellIndex,
globalCellIndex + 1
};
if (periodic)
{
if (globalCellIndex == 0)
{
(new CellFaceTag()
{
EdgeTag = periodicTag,
PeriodicInverse = true,
ConformalNeighborship = true,
FaceIndex = (int)Line.Edge.Left,
NeighCell_GlobalID = globalNumberOfCells - 1
}).AddToArray(ref cell.CellFaceTags);
}
else if (globalCellIndex == globalNumberOfCells - 1)
{
(new CellFaceTag()
{
EdgeTag = periodicTag,
PeriodicInverse = false,
ConformalNeighborship = true,
FaceIndex = (int)Line.Edge.Right,
NeighCell_GlobalID = 0
}).AddToArray(ref cell.CellFaceTags);
}
}
grid.Cells[localCellIndex] = cell;
localCellIndex++;
}
}
return(grid);
}
}
/// <summary>
/// Polynomial extrapolation between cells.
/// </summary>
/// <param name="ExtrapolateTo">contains all cells for which extrapolated values should be computed</param>
/// <param name="ExtrapolateFrom">contains all cells with "known values"</param>
/// <returns>
/// The number of cells (locally) for which the algorithm was not able to extrapolate a value
/// </returns>
virtual public int CellExtrapolation(CellMask ExtrapolateTo, CellMask ExtrapolateFrom)
{
MPICollectiveWatchDog.Watch();
int J = this.GridDat.iLogicalCells.NoOfLocalUpdatedCells;
int NoOfNeigh;
int[] NeighIdx;
int[][] CN = this.GridDat.iLogicalCells.CellNeighbours;
// mark all cells in which species 'Id' is known
// ---------------------------------------------
BitArray ValueIsKnown = ExtrapolateFrom.GetBitMaskWithExternal().CloneAs();
CellMask _ExtrapolateTo = ExtrapolateTo.Except(ExtrapolateFrom);
BitArray NeedToKnowValue = _ExtrapolateTo.GetBitMask();
this.Clear(_ExtrapolateTo);
// repeat until (for species 'Id' the DOF' of) all cells
// that contain (at least a fraction of) species 'Id' are known...
// ------------------------------------------------------------------
int NoOfCells_ToExtrapolate = _ExtrapolateTo.NoOfItemsLocally;
int NoOfCells_ExtrapolatedInSweep = 1;
int sweepcnt = 0;
List <double> scaling = new List <double>();
List <int[]> CellPairs = new List <int[]>();
BitArray cells_mod_in_sweep = new BitArray(J);
while (true)
{
// MPI-parallel evaluation of termination criterion
// ------------------------------------------------
int bool_LocalRun = ((NoOfCells_ToExtrapolate > 0) && (NoOfCells_ExtrapolatedInSweep > 0)) ? 1 : 0;
int bool_GlobalRun = 0;
unsafe
{
csMPI.Raw.Allreduce((IntPtr)(&bool_LocalRun), (IntPtr)(&bool_GlobalRun), 1, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.MAX, csMPI.Raw._COMM.WORLD);
}
if (bool_GlobalRun <= 0)
{
// finished on all MPI processors
break;
}
// MPI-update before local sweep: necessary for consistent result of alg.
this.MPIExchange();
if (sweepcnt > 0)
{
ValueIsKnown.MPIExchange(this.GridDat);
}
// local work
// ----------
if (bool_LocalRun > 0)
{
sweepcnt++;
NoOfCells_ExtrapolatedInSweep = 0;
scaling.Clear();
CellPairs.Clear();
cells_mod_in_sweep.SetAll(false);
for (int j = 0; j < J; j++)
{
// determine whether there is something to do for cell 'j' or not ...
bool needToKnowSpecies = NeedToKnowValue[j];
bool _mustbeExtrapolated = needToKnowSpecies && !ValueIsKnown[j];
if (_mustbeExtrapolated)
{
// continuation for this cell is needed
// ++++++++++++++++++++++++++++++++++++
// try to find a neighbour
NeighIdx = CN[j].CloneAs();
NoOfNeigh = NeighIdx.Length;
int FoundNeighs = 0;
for (int nn = 0; nn < NoOfNeigh; nn++)
{
if (ValueIsKnown[NeighIdx[nn]])
{
// bingo
FoundNeighs++;
}
else
{
NeighIdx[nn] = -1; // not usable
}
}
if (FoundNeighs <= 0)
{
// hope for better luck in next sweep
continue;
}
//Array.Clear(u2, 0, N);
for (int nn = 0; nn < NoOfNeigh; nn++)
{
if (NeighIdx[nn] < 0)
{
continue;
}
int _2 = j; // cell to extrapolate TO
int _1 = NeighIdx[nn]; // cell to extrapolate FROM
CellPairs.Add(new int[] { _1, _2 });
double ooFoundNeighs = 1.0 / (double)FoundNeighs;
scaling.Add(ooFoundNeighs);
}
cells_mod_in_sweep[j] = true;
NoOfCells_ExtrapolatedInSweep++;
}
}
int E = CellPairs.Count;
int[,] _CellPairs = new int[E, 2];
for (int e = 0; e < E; e++)
{
_CellPairs.SetRow(e, CellPairs[e]);
}
double[] preScale = new double[scaling.Count];
preScale.SetAll(1.0);
this.CellExtrapolation(_CellPairs, scaling, preScale);
for (int j = 0; j < J; j++)
{
if (cells_mod_in_sweep[j] == true)
{
ValueIsKnown[j] = true;
}
}
NoOfCells_ToExtrapolate -= NoOfCells_ExtrapolatedInSweep;
}
}
// return
// ------
return(NoOfCells_ToExtrapolate);
}
/// <summary>
/// Like <see cref="Evaluate(double, IEnumerable{DGField}, MultidimensionalArray, double, MultidimensionalArray, BitArray, int[])"/>,
/// but with MPI-Exchange
/// </summary>
public int EvaluateParallel(double alpha, IEnumerable <DGField> Flds, MultidimensionalArray Points, double beta, MultidimensionalArray Result, BitArray UnlocatedPoints = null)
{
using (new FuncTrace()) {
MPICollectiveWatchDog.Watch();
int L = Points != null ? Points.NoOfRows : 0;
int MPIsize = m_Context.MpiSize;
int D = m_Context.SpatialDimension;
if (UnlocatedPoints != null)
{
if (UnlocatedPoints.Length != L)
{
throw new ArgumentException("Length mismatch");
}
}
// evaluate locally
// ================
var unlocated = new System.Collections.BitArray(L);
int NoOfUnlocated;
if (L > 0)
{
NoOfUnlocated = this.Evaluate(alpha, Flds, Points, beta, Result, unlocated);
}
else
{
NoOfUnlocated = 0;
}
// return, if there are no unlocalized points
// ==========================================
int TotNoOfUnlocated = NoOfUnlocated.MPISum();
if (TotNoOfUnlocated <= 0)
{
if (UnlocatedPoints != null)
{
UnlocatedPoints.SetAll(false);
}
return(0);
}
// copy unlocalized to separate array
// ==================================
double[,] localUnlocated = new double[NoOfUnlocated, D]; // MultidimensionalArray does not allow zero length -- so use double[,] instead
int[] IndexToOrgIndex = new int[NoOfUnlocated];
int u = 0;
for (int i = 0; i < L; i++)
{
if (unlocated[i])
{
localUnlocated.SetRowPt(u, Points.GetRowPt(i));
IndexToOrgIndex[u] = i;
u++;
}
}
Debug.Assert(u == NoOfUnlocated);
// collect on all ranks -- this won't scale well, but it may work
// ==============================================================
MultidimensionalArray globalUnlocated;
int[] WhoIsInterestedIn; // index: point index, corresponds with 'globalUnlocated' rows; content: rank which needs the result
int[] OriginalIndex; // index: detto; content: index which the point had on the processor that sent it.
double[][,] __globalUnlocated;
int LL;
{
__globalUnlocated = localUnlocated.MPIAllGatherO();
Debug.Assert(__globalUnlocated.Length == MPIsize);
Debug.Assert(__globalUnlocated.Select(aa => aa.GetLength(0)).Sum() == TotNoOfUnlocated);
Debug.Assert(__globalUnlocated[m_Context.MpiRank].GetLength(0) == NoOfUnlocated);
LL = TotNoOfUnlocated - NoOfUnlocated;
if (LL > 0)
{
globalUnlocated = MultidimensionalArray.Create(LL, D);
}
else
{
globalUnlocated = null;
}
WhoIsInterestedIn = new int[LL];
OriginalIndex = new int[LL];
int g = 0;
for (int r = 0; r < MPIsize; r++) // concat all point arrays from all processors
{
if (r == m_Context.MpiRank)
{
continue;
}
double[,] __globalPart = __globalUnlocated[r];
int Lr = __globalPart.GetLength(0);
if (Lr > 0)
{
globalUnlocated.ExtractSubArrayShallow(new[] { g, 0 }, new[] { g + Lr - 1, D - 1 }).Acc2DArray(1.0, __globalPart);
}
for (int i = 0; i < Lr; i++)
{
WhoIsInterestedIn[i + g] = r;
OriginalIndex[i + g] = i;
}
g += Lr;
}
}
// try to evaluate the so-far-unlocalized points
// ---------------------------------------------
var unlocated2 = new System.Collections.BitArray(LL);
var Result2 = LL > 0 ? MultidimensionalArray.Create(LL, Flds.Count()) : null;
int NoOfUnlocated2 = LL > 0 ? this.Evaluate(1.0, Flds, globalUnlocated, 0.0, Result2, unlocated2) : 0;
// backward MPI sending
// --------------------
IDictionary <int, EvaluateParallelHelper> resultFromOtherProcs;
{
var backSend = new Dictionary <int, EvaluateParallelHelper>();
for (int ll = 0; ll < LL; ll++)
{
if (!unlocated2[ll])
{
int iTarget = WhoIsInterestedIn[ll];
Debug.Assert(iTarget != m_Context.MpiRank);
if (!backSend.TryGetValue(iTarget, out EvaluateParallelHelper eph))
{
eph = new EvaluateParallelHelper();
backSend.Add(iTarget, eph);
}
eph.OriginalIndices.Add(OriginalIndex[ll]);
eph.Results.Add(Result2.GetRow(ll));
}
}
resultFromOtherProcs = SerialisationMessenger.ExchangeData(backSend);
}
// fill the results from other processors
// ======================================
foreach (var res in resultFromOtherProcs.Values)
{
int K = res.OriginalIndices.Count();
Debug.Assert(res.OriginalIndices.Count == res.Results.Count);
for (int k = 0; k < K; k++)
{
int iOrg = IndexToOrgIndex[res.OriginalIndices[k]];
if (unlocated[iOrg] == true)
{
NoOfUnlocated--;
}
unlocated[iOrg] = false;
Result.AccRow(iOrg, alpha, res.Results[k]);
}
}
// Return
// ======
if (UnlocatedPoints != null)
{
for (int l = 0; l < L; l++)
{
UnlocatedPoints[l] = unlocated[l];
}
}
return(NoOfUnlocated);
}
}
/// <summary>
/// constructor
/// </summary>
protected Solver()
{
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
}
/// <summary>
/// ctor.
/// </summary>
internal XQuadSchemeHelper(XDGSpaceMetrics __XDGSpaceMetrics)
{
MPICollectiveWatchDog.Watch();
this.XDGSpaceMetrics = __XDGSpaceMetrics;
ConstructorCommon();
}
/// <summary>
/// creates a HYPRE solver
/// </summary>
public Solver()
{
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
CreateSolver();
}
/// <summary>
/// ctor
/// </summary>
/// <param name="volMask">
/// volume mask for those cells which should be contained in the subgrid
/// </param>
public SubGrid(CellMask volMask)
{
MPICollectiveWatchDog.Watch();
this.m_VolumeMask = volMask;
m_GridData = volMask.GridData;
}
/// <summary>
/// Returns the minimum and the maximum value of the DG field
/// This is a collective call, it must be invoked by all
/// MPI processes within the communicator; internally, it invokes MPI_Allreduce;
/// </summary>
/// <param name="max">
/// on all invoking MPI processes, the maximum value (over all processors) of
/// this field;
/// </param>
/// <param name="min">
/// on all invoking MPI processes, the minimum value (over all processors) of
/// this field;
/// </param>
/// <remarks>
/// to find the maximum value, this field is evaluated on each vertex and the center of the simplex.
/// </remarks>
/// <param name="cm">
/// optional domain restriction
/// </param>
/// <param name="jMaxGlob">
/// global cell index in which the maximum value <paramref name="max"/> was found
/// </param>
/// <param name="jMinGlob">
/// global cell index in which the maximum value <paramref name="min"/> was found
/// </param>
public void GetExtremalValues(out double min, out double max, out int jMinGlob, out int jMaxGlob, CellMask cm = null)
{
MPICollectiveWatchDog.Watch(csMPI.Raw._COMM.WORLD);
using (new FuncTrace()) {
int J = Basis.GridDat.iLogicalCells.NoOfLocalUpdatedCells;
// find maximum on this processor
// ------------------------------
// create node set
InitExtremalProbeNS();
// vectorisation of this method
int VectorSize = -1;
int N0 = m_ExtremalProbeNS[0].GetLength(0); // number of nodes
MultidimensionalArray fieldValues = new MultidimensionalArray(2);
IEnumerable <Chunk> all_chunks;
if (cm == null)
{
var _ch = new Chunk[1];
_ch[0].i0 = 0;
_ch[0].Len = J;
all_chunks = _ch;
}
else
{
all_chunks = cm;
}
double LocMax = -double.MaxValue, LocMin = double.MaxValue;
int jMinLoc = int.MaxValue, jMaxLoc = int.MaxValue;
foreach (Chunk chk in all_chunks)
{
VectorSize = this.GridDat.iGeomCells.GetNoOfSimilarConsecutiveCells(CellInfo.RefElementIndex_Mask, chk.i0, Math.Min(100, chk.Len)); // try to be a little vectorized;
int _J = chk.Len + chk.i0;
for (int j = chk.i0; j < _J; j += VectorSize)
{
if (j + VectorSize > _J)
{
VectorSize = _J - j;
}
int iKref = Basis.GridDat.iGeomCells.GetRefElementIndex(j);
int N = m_ExtremalProbeNS[iKref].GetLength(0);
if (fieldValues.GetLength(0) != VectorSize || fieldValues.GetLength(1) != N)
{
fieldValues = MultidimensionalArray.Create(VectorSize, N);
}
this.Evaluate(j, VectorSize, m_ExtremalProbeNS[iKref], fieldValues, 0.0);
// loop over cells ...
for (int jj = j; jj < j + VectorSize; jj++)
{
// loop over nodes ...
for (int n = 0; n < N; n++)
{
double vel = 0;
vel = fieldValues[jj - j, n];
if (vel > LocMax)
{
LocMax = vel;
jMaxLoc = jj;
}
if (vel < LocMin)
{
LocMin = vel;
jMinLoc = jj;
}
}
}
}
}
// find the maximum over all processes via MPI and return
// ------------------------------------------------------
if (Basis.GridDat.CellPartitioning.MpiSize > 1)
{
double[] total = new double[2];
double[] local = new double[] { LocMax, -LocMin };
unsafe
{
fixed(double *ploc = local, ptot = total)
{
csMPI.Raw.Allreduce((IntPtr)ploc, (IntPtr)ptot, 2, csMPI.Raw._DATATYPE.DOUBLE, csMPI.Raw._OP.MAX, csMPI.Raw._COMM.WORLD);
}
}
max = total[0];
min = -total[1];
int i0 = Basis.GridDat.CellPartitioning.i0;
int[] jGlob = new int[2];
if (max == LocMax)
{
// (at least one) global maximum on this processor
jGlob[0] = jMaxLoc + i0;
}
else
{
// maximum on some other processor
jGlob[0] = int.MaxValue;
}
if (min == LocMin)
{
// (at least one) global minimum on this processor
jGlob[1] = jMinLoc + i0;
}
else
{
// minimum on some other processor
jGlob[1] = int.MaxValue;
}
// in case of multiple global minimums/maximums, e.g. for a constant field, we return the lowest (jMaxGlob,jMinGlob)
int[] jGlobM = new int[2];
unsafe
{
fixed(int *ploc = jGlob, ptot = jGlobM)
{
csMPI.Raw.Allreduce((IntPtr)ploc, (IntPtr)ptot, 2, csMPI.Raw._DATATYPE.INT, csMPI.Raw._OP.MIN, csMPI.Raw._COMM.WORLD);
}
}
jMaxGlob = jGlob[0];
jMinGlob = jGlob[1];
}
else
{
min = LocMin;
max = LocMax;
jMaxGlob = jMaxLoc;
jMinGlob = jMinLoc;
}
}
}
