static bool AreBasicPropertiesEqual(GridCommons A, GridCommons B)
{
if (object.ReferenceEquals(A, B))
{
return(true);
}
if ((A == null) != (B == null))
{
return(false);
}
ilPSP.MPICollectiveWatchDog.Watch(MPI.Wrappers.csMPI.Raw._COMM.WORLD);
int glbNoOfCells_A = A.NumberOfCells;
int glbNoOfCells_B = B.NumberOfCells;
int glbNoOfBcCells_A = A.NumberOfBcCells;
int glbNoOfBcCells_B = B.NumberOfBcCells;
if (glbNoOfCells_A != glbNoOfCells_B)
{
return(false);
}
if (glbNoOfBcCells_A != glbNoOfBcCells_B)
{
return(false);
}
if (!ArrayTools.ListEquals(A.RefElements, B.RefElements, (a, b) => object.ReferenceEquals(a, b)))
{
return(false);
}
if (!ArrayTools.ListEquals(A.EdgeRefElements, B.EdgeRefElements, (a, b) => object.ReferenceEquals(a, b)))
{
return(false);
}
if (!A.EdgeTagNames.Keys.SetEquals(B.EdgeTagNames.Keys))
{
return(false);
}
foreach (var nmn in A.EdgeTagNames.Keys)
{
if (A.EdgeTagNames[nmn] != B.EdgeTagNames[nmn])
{
return(false);
}
}
if (!ArrayTools.ListEquals(A.PeriodicTrafo, B.PeriodicTrafo, (a, b) => a.ApproximateEquals(b)))
{
return(false);
}
return(true);
}
static bool AreReferencesEqual(GridCommons A, GridCommons B)
{
if (object.ReferenceEquals(A, B))
{
return(true);
}
if ((A == null) != (B == null))
{
return(false);
}
ilPSP.MPICollectiveWatchDog.Watch(MPI.Wrappers.csMPI.Raw._COMM.WORLD);
int glbNoOfCells_A = A.NumberOfCells;
int glbNoOfCells_B = B.NumberOfCells;
int glbNoOfBcCells_A = A.NumberOfBcCells;
int glbNoOfBcCells_B = B.NumberOfBcCells;
if (glbNoOfCells_A != glbNoOfCells_B)
{
return(false);
}
if (glbNoOfBcCells_A != glbNoOfBcCells_B)
{
return(false);
}
if (!ArrayTools.ListEquals(A.RefElements, B.RefElements, (a, b) => object.ReferenceEquals(a, b)))
{
return(false);
}
if (!ArrayTools.ListEquals(A.EdgeRefElements, B.EdgeRefElements, (a, b) => object.ReferenceEquals(a, b)))
{
return(false);
}
if (!ArrayTools.ListEquals(A.EdgeTagNames, B.EdgeTagNames, (a, b) => (a.Key == b.Key && a.Value.Equals(b.Value))))
{
return(false);
}
if (!ArrayTools.ListEquals(A.PeriodicTrafo, B.PeriodicTrafo, (a, b) => a.ApproximateEquals(b)))
{
return(false);
}
return(true);
}
/// <summary>
/// L2-norm over all entries
/// </summary>
static public double L2Dist(this MultidimensionalArray mda, MultidimensionalArray mdb)
{
if (!ArrayTools.ListEquals(mda.Lengths, mdb.Lengths, (La, Lb) => La == Lb))
{
throw new ArgumentException("Arrays must have the same length.");
}
double ret = 0;
mda.ApplyAll(delegate(int[] idx, double entry_a) {
double entry_b = mdb[idx];
double dist = entry_a - entry_b;
ret += dist * dist;
});
return(Math.Sqrt(ret));
}
/// <summary>
/// Start-indices of sub-blocks.
/// </summary>
public int[] GetSubblk_i0(int blockType)
{
#if DEBUG
// make sure that no one outside modifies this member
if (m_Subblki0_Backup == null)
{
m_Subblki0_Backup = m_Subblk_i0.Select(bl => (int[])(bl.Clone())).ToArray();
}
else
{
Debug.Assert(m_Subblk_i0.Length == m_Subblki0_Backup.Length);
for (int i = 0; i < m_SubblkLen.Length; i++)
{
Debug.Assert(ArrayTools.ListEquals(m_Subblk_i0[i], m_Subblki0_Backup[i]), "Some moron messed with blocking indices.");
}
}
#endif
return(m_Subblk_i0[blockType]);
}
public int[] GetSubblk_i0(int blockType)
{
if (m_j0Subblk_i0 == null)
{
m_j0Subblk_i0 = ArrayTools.GetSubVector(this.m_j0CoordinateIndex, 0, this.m_BasisS.Length);
}
#if DEBUG
if (m_j0Subblk_i0_Backup == null)
{
m_j0Subblk_i0_Backup = m_j0Subblk_i0.CloneAs();
}
else
{
Debug.Assert(ArrayTools.ListEquals <int>(m_j0Subblk_i0, m_j0Subblk_i0_Backup), "Some moron messed with the start indices.");
}
#endif
return(m_j0Subblk_i0);
}
Dictionary <SpeciesId, MassMatrixFactory.MassMatrixBlockContainer>[] GetInverseMassMatrixBlocks(IEnumerable <SpeciesId> RequestedSpecies, int[] Degrees)
{
int[] Ns = Degrees.Select(p => this.MaxBasis.Polynomials[0].Where(poly => poly.AbsoluteDegree <= p).Count()).ToArray();
for (int iKref = 1; iKref < this.MaxBasis.Polynomials.Count; iKref++)
{
int[] _Ns = Degrees.Select(p => this.MaxBasis.Polynomials[0].Where(poly => poly.AbsoluteDegree <= p).Count()).ToArray();
if (!ArrayTools.ListEquals(Ns, _Ns))
{
throw new NotSupportedException();
}
}
Dictionary <SpeciesId, MassMatrixFactory.MassMatrixBlockContainer>[] InvMassBlocks = new Dictionary <SpeciesId, MassMatrixBlockContainer> [Degrees.Length];
for (int i = 0; i < Degrees.Length; i++)
{
int pSame = -1;
//pSame = Array.IndexOf(Degrees, Degrees[i], 0, i);
for (int j = 0; j < i; j++)
{
if ((Degrees[j] == Degrees[i])
//&& (VariableAgglomerationSwitch[j] == VariableAgglomerationSwitch[i])
)
{
pSame = j;
break;
}
}
if (pSame >= 0)
{
InvMassBlocks[i] = InvMassBlocks[pSame];
}
else
{
InvertMassMatrixBlocks(out InvMassBlocks[i], this.MassBlocks, Ns[i]);
}
}
return(InvMassBlocks);
}
Dictionary <Tuple <SpeciesId[], XQuadFactoryHelper.MomentFittingVariants, int>, XDGSpaceMetrics> NewXDGSpaceMetricsCache()
{
return(new Dictionary <Tuple <SpeciesId[], XQuadFactoryHelper.MomentFittingVariants, int>, XDGSpaceMetrics>(
new FuncEqualityComparer <Tuple <SpeciesId[], XQuadFactoryHelper.MomentFittingVariants, int> >(
delegate(Tuple <SpeciesId[], XQuadFactoryHelper.MomentFittingVariants, int> A, Tuple <SpeciesId[], XQuadFactoryHelper.MomentFittingVariants, int> B) {
//if(.)
if (!ArrayTools.ListEquals(A.Item1, B.Item1, (a, b) => a.cntnt == b.cntnt))
{
return false;
}
if (A.Item2 != B.Item2)
{
return false;
}
if (A.Item3 != B.Item3)
{
return false;
}
return true;
})));
}
static bool AreCellsEqual(GridCommons A, GridCommons B)
{
if (object.ReferenceEquals(A, B))
{
return(true);
}
if ((A == null) != (B == null))
{
return(false);
}
if (A.Cells == null)
{
throw new ArgumentException();
}
int A_NumberOfBcCells = A.NumberOfBcCells;
int match = 1;
{
// load cells of grid B, if required
// ---------------------------------
Cell[] B_Cells;
if (B.Cells == null)
{
throw new Exception("Cells are not initialized");
}
else
{
B_Cells = B.Cells;
}
if (A.Cells.Length != B_Cells.Length)
{
throw new ApplicationException();
}
// put the cells of B into the same order as those of A
// ----------------------------------------------------
{
// tau is the GlobalID-permutation that we have for the loaded vector
// sigma is the current GlobalID-permutation of the grid
var sigma = new Permutation(A.Cells.Select(cell => cell.GlobalID).ToArray(), csMPI.Raw._COMM.WORLD);
var tau = new Permutation(B_Cells.Select(cell => cell.GlobalID).ToArray(), csMPI.Raw._COMM.WORLD);
if (sigma.TotalLength != tau.TotalLength)
{
// should have been checked already
throw new ArgumentException();
}
// compute resorting permutation
Permutation invSigma = sigma.Invert();
Permutation Resorting = invSigma * tau;
tau = null; // Werfen wir sie dem GC zum Fraße vor!
invSigma = null;
// put dg coordinates into right order
Resorting.ApplyToVector(B_Cells.CloneAs(), B_Cells);
}
// compare cells
// -------------
for (int j = 0; j < A.Cells.Length; j++)
{
Cell Ca = A.Cells[j];
Cell Cb = B_Cells[j];
Debug.Assert(Ca.GlobalID == Cb.GlobalID);
if (!ArrayTools.ListEquals(Ca.NodeIndices, Cb.NodeIndices, (ia, ib) => ia == ib))
{
match = 0;
break;
}
if (Ca.Type != Cb.Type)
{
match = 0;
break;
}
if (Ca.CellFaceTags != null || Cb.CellFaceTags != null)
{
CellFaceTag[] CFTA = Ca.CellFaceTags != null ? Ca.CellFaceTags : new CellFaceTag[0];
CellFaceTag[] CFTB = Cb.CellFaceTags != null ? Cb.CellFaceTags : new CellFaceTag[0];
if (CFTA.Length != CFTB.Length)
{
match = 0;
break;
}
bool setMatch = true;
for (int i1 = 0; i1 < CFTA.Length; i1++)
{
bool b = false;
for (int j1 = 0; j1 < CFTB.Length; j1++)
{
if (CFTA[i1].Equals(CFTB[j1]))
{
b = true;
break;
}
}
if (b == false)
{
setMatch = false;
break;
}
}
if (!setMatch)
{
match = 0;
break;
}
}
double h = Math.Min(Ca.TransformationParams.MindistBetweenRows(), Cb.TransformationParams.MindistBetweenRows());
double L2Dist = Ca.TransformationParams.L2Dist(Cb.TransformationParams);
if (L2Dist > h * 1.0e-9)
{
match = 0;
break;
}
}
}
if (A_NumberOfBcCells > 0)
{
BCElement[] B_BcCells;
if (B.BcCells == null && !B.BcCellsStorageGuid.Equals(Guid.Empty))
{
throw new Exception("Bc Cells are not initialized");
}
else
{
B_BcCells = B.BcCells;
}
if (A.BcCells.Length != B_BcCells.Length)
{
throw new ApplicationException("Internal error.");
}
// put the cells of B into the same order as those of A
// ----------------------------------------------------
{
long Offset = A.NumberOfCells_l;
// tau is the GlobalID-permutation that we have for the loaded vector
// sigma is the current GlobalID-permutation of the grid
var sigma = new Permutation(A.BcCells.Select(cell => cell.GlobalID - Offset).ToArray(), csMPI.Raw._COMM.WORLD);
var tau = new Permutation(B_BcCells.Select(cell => cell.GlobalID - Offset).ToArray(), csMPI.Raw._COMM.WORLD);
if (sigma.TotalLength != tau.TotalLength)
{
// should have been checked already
throw new ArgumentException();
}
// compute resorting permutation
Permutation invSigma = sigma.Invert();
Permutation Resorting = invSigma * tau;
tau = null; // Werfen wir sie dem GC zum Fraße vor!
invSigma = null;
// put dg coordinates into right order
Resorting.ApplyToVector(B_BcCells.CloneAs(), B_BcCells);
}
// compare cells
// -------------
for (int j = 0; j < A.BcCells.Length; j++)
{
BCElement Ca = A.BcCells[j];
BCElement Cb = B_BcCells[j];
Debug.Assert(Ca.GlobalID == Cb.GlobalID);
if (!ArrayTools.ListEquals(Ca.NodeIndices, Cb.NodeIndices, (ia, ib) => ia == ib))
{
match = 0;
break;
}
if (Ca.Type != Cb.Type)
{
match = 0;
break;
}
if (Ca.Conformal != Cb.Conformal)
{
match = 0;
break;
}
if (Ca.EdgeTag != Cb.EdgeTag)
{
match = 0;
break;
}
if (Ca.NeighCell_GlobalIDs != null || Cb.NeighCell_GlobalIDs != null)
{
long[] NgA = Ca.NeighCell_GlobalIDs != null ? Ca.NeighCell_GlobalIDs : new long[0];
long[] NgB = Cb.NeighCell_GlobalIDs != null ? Cb.NeighCell_GlobalIDs : new long[0];
if (NgA.Length != NgB.Length)
{
match = 0;
break;
}
bool setMatch = true;
for (int i1 = 0; i1 < NgA.Length; i1++)
{
bool b = false;
for (int j1 = 0; j1 < NgB.Length; j1++)
{
if (NgA[i1] == NgB[j1])
{
b = true;
break;
}
}
if (b == false)
{
setMatch = false;
break;
}
}
if (!setMatch)
{
match = 0;
break;
}
}
double h = Math.Min(Ca.TransformationParams.MindistBetweenRows(), Cb.TransformationParams.MindistBetweenRows());
double L2Dist = Ca.TransformationParams.L2Dist(Cb.TransformationParams);
if (L2Dist > h * 1.0e-9)
{
match = 0;
break;
}
}
}
match = match.MPIMin();
return(match > 0);
}
/// <summary>
/// computes the injector for multigrid level 1
/// </summary>
/// <seealso cref="BuildInjector_Lv2andup"/>
private static MultidimensionalArray[] BuildInjector_Lv1(
Basis maxDgBasis, int Np,
MultidimensionalArray InjectorsBase, bool[] InjectorsBaseReady,
int Jagg, int[][] Ag2Pt, int[][] C2F)
{
using (new FuncTrace()) {
MultidimensionalArray ortho = MultidimensionalArray.Create(Np, Np);
MultidimensionalArray[] Injectors_iLevel = new MultidimensionalArray[Jagg];
#if DEBUG
{
int Jbase = InjectorsBase.GetLength(0);
if (InjectorsBase.GetLength(1) != InjectorsBase.GetLength(2))
{
throw new ArgumentException();
}
int N = InjectorsBase.GetLength(1);
var check = MultidimensionalArray.Create(N, N);
for (int j = 0; j < Jbase; j++)
{
check.Clear();
check.AccEye(1.0);
check.Acc(-1.0, InjectorsBase.ExtractSubArrayShallow(j, -1, -1));
if (check.InfNorm() != 0.0)
{
throw new ArgumentException();
}
}
}
#endif
var iLPar = maxDgBasis.GridDat.iLogicalCells; // cells of *parent* grid
for (int j = 0; j < Jagg; j++) // loop over aggregate cells
{
Debug.Assert(ArrayTools.ListEquals(Ag2Pt[j], C2F[j]));
int[] compCell = Ag2Pt[j];
int I = compCell.Length;
int iRoot = -1;
double maxSize = -1.0;
for (int i = 0; i < I; i++)
{
double sz = iLPar.GetCellVolume(compCell[i]);
if (sz <= 0.0)
{
throw new ArithmeticException("found cell with non-positive volume.");
}
if (sz > maxSize)
{
iRoot = i;
maxSize = sz;
}
}
//var gdat = ((maxDgBasis.GridDat) as Classic.GridData);
//double[] Sizes = compCell.Select(jPart => gdat.iLogicalCells.GetCellVolume(jPart)).ToArray();
Injectors_iLevel[j] = MultidimensionalArray.Create(I, Np, Np);
if (I > 1)
{
// compute extrapolation
int[,] CellPairs = new int[I - 1, 2];
int cnt = 0;
for (int i = 0; i < I; i++)
{
if (i != iRoot)
{
CellPairs[cnt, 0] = compCell[iRoot];
CellPairs[cnt, 1] = compCell[i];
Debug.Assert(CellPairs[cnt, 0] != CellPairs[cnt, 1]);
cnt++;
}
}
//cnt = 0;
//for (int i = 0; i < I - 1; i++) {
// CellPairs[cnt, 0] = compCell[0];
// CellPairs[cnt, 1] = compCell[i + 1];
// cnt++;
//}
var ExpolMtx = MultidimensionalArray.Create(I, Np, Np);
maxDgBasis.GetExtrapolationMatrices(CellPairs, ExpolMtx.ExtractSubArrayShallow(new int[] { 1, 0, 0 }, new int[] { I - 1, Np - 1, Np - 1 }));
for (int i = 0; i < iRoot; i++)
{
var M2 = ExpolMtx.ExtractSubArrayShallow(i, -1, -1);
var M1 = ExpolMtx.ExtractSubArrayShallow(i + 1, -1, -1);
M2.Set(M1);
}
for (int n = 0; n < Np; n++)
{
for (int m = 0; m < Np; m++)
{
ExpolMtx[iRoot, n, m] = n == m ? 1.0 : 0.0;
}
}
// Compute intermediate mass matrix
var MMtemp = MultidimensionalArray.Create(Np, Np);
MMtemp.Multiply(1.0, ExpolMtx, ExpolMtx, 0.0, "nm", "iln", "ilm");
// orthonormalize
//try {
MMtemp.SymmetricLDLInversion(ortho, null); // ortho is output, will be overwritten
//} catch (ArithmeticException ae) {
// PlotScheisse(gdat.Grid, compCell);
//}
Injectors_iLevel[j].Multiply(1.0, ExpolMtx, ortho, 0.0, "inm", "ink", "km");
}
else
{
Injectors_iLevel[j].ExtractSubArrayShallow(0, -1, -1).AccEye(1.0);
}
// base level injector
var injBase = InjectorsBase.ExtractSubArrayShallow(compCell[0], -1, -1);
injBase.Set(Injectors_iLevel[j].ExtractSubArrayShallow(0, -1, -1));
Debug.Assert(InjectorsBaseReady[compCell[0]]);
for (int i = 1; i < I; i++)
{
InjectorsBaseReady[compCell[i]] = false;
}
}
return(Injectors_iLevel);
}
}
/// <summary>
/// only executed on proc 0
/// </summary>
bool PARDISOInitAndSolve(IMutableMatrixEx Mtx, double[] _x, double[] _b)
{
using (new FuncTrace()) {
if (m_PardisoMatrix == null)
{
m_PardisoMatrix = new Matrix(Mtx);
}
int rank;
csMPI.Raw.Comm_Rank(csMPI.Raw._COMM.WORLD, out rank);
if (rank == 0)
{
#if DEBUG
double[] aClone = (double[])m_PardisoMatrix.a.Clone();
int[] iaClone = (int[])m_PardisoMatrix.ia.Clone();
int[] jaClone = (int[])m_PardisoMatrix.ja.Clone();
#endif
unsafe
{
fixed(double *dparam = m_PardInt.m_dparam)
{
fixed(double *a = m_PardisoMatrix.a, x = _x, b = _b)
{
fixed(int *ia = m_PardisoMatrix.ia, ja = m_PardisoMatrix.ja, iparm = m_PardInt.m_parm, __pt = m_PardInt.m_pt)
{
int n = m_PardisoMatrix.n;
//int nnz = ia[n];
int mtype = GetMType();
int nrhs = 1; /* Number of right hand sides. */
/* Internal solver memory pointer pt, */
/* 32-bit: int pt[64]; 64-bit: long int pt[64] */
/* or void *pt[64] should be OK on both architectures */
void *pt = (void *)__pt;
/* Pardiso control parameters. */
int maxfct = m_PardInt.maxfct, mnum = m_PardInt.mnum, msglvl = m_PardInt.msglvl;
int phase, error;
/* Auxiliary variables. */
//int i;
double ddum; /* Double dummy */
int idum; /* Integer dummy. */
if (!m_PardInt.m_PardisoInitialized)
{
/* -------------------------------------------------------------------- */
/* .. Setup Pardiso control parameters und initialize the solvers */
/* internal adress pointers. This is only necessary for the FIRST */
/* call of the PARDISO solver. */
/* ---------------------------------------------------------------------*/
if (this.Version == PARDISO.Version.MKL)
{
// Do nothing NUM_THREADS is set by environment variables in MKLPardiso
// iparm[2] stays set to zero
// If you want to control this manually, do something like this:
//System.Environment.SetEnvironmentVariable("OMP_NUM_THREADS", "4");
//System.Environment.SetEnvironmentVariable("MKL_NUM_THREADS", "4");
//If this is true: MKL determines the number of OMP threads automatically, based on proc information
//System.Environment.SetEnvironmentVariable("MKL_DYNAMIC", "false");
}
else if ((this.Version == PARDISO.Version.v4) || (this.Version == PARDISO.Version.v5))
{
// Get Value for IPARM(3) from the Environment Variable
int NumOfProcs = Convert.ToInt32(System.Environment.GetEnvironmentVariable("OMP_NUM_THREADS"));
iparm[2] = NumOfProcs;
#if DEBUG
Console.WriteLine("IPARM(3) - NumberOfProcessors set to {0}", iparm[2]);
#endif
}
wrapper.PARDISOINIT(pt, &mtype, iparm, dparam);
//Console.WriteLine("init: IPARAM(22) = {0}, IPARAM(23) = {1}", iparm[21], iparm[22]);
maxfct = 1; /* Maximum number of numerical factorizations. */
mnum = 1; /* Which factorization to use. */
msglvl = 0; /* Print statistical information */
error = 0; /* Initialize error flag */
/* -------------------------------------------------------------------- */
/* .. Reordering and Symbolic Factorization. This step also allocates */
/* all memory that is necessary for the factorization. */
/* -------------------------------------------------------------------- */
phase = 11;
iparm[59] = 0;
//Console.Write("calling pardiso, phase 11... ");
wrapper.PARDISO(pt, &maxfct, &mnum, &mtype, &phase,
&n, a, ia, ja, &idum, &nrhs,
iparm, &msglvl, &ddum, &ddum, &error, dparam);
//Console.WriteLine("11: IPARAM(22) = {0}, IPARAM(23) = {1}", iparm[21], iparm[22]);
if (error != 0)
{
PARDISODispose();
Console.WriteLine("PARDISO ERROR: " + wrapper.PARDISOerror2string(error));
return(false);
}
//Console.Write("\nReordering completed ... ");
//Console.Write("\nNumber of nonzeros in factors = %d", iparm[17]);
//Console.Write("\nNumber of factorization MFLOPS = %d", iparm[18]);
/* -------------------------------------------------------------------- */
/* .. Numerical factorization. */
/* -------------------------------------------------------------------- */
phase = 22;
wrapper.PARDISO(pt, &maxfct, &mnum, &mtype, &phase,
&n, a, ia, ja, &idum, &nrhs,
iparm, &msglvl, &ddum, &ddum, &error, dparam);
//Console.WriteLine("22: IPARAM(22) = {0}, IPARAM(23) = {1}", iparm[21], iparm[22]);
if (error != 0)
{
PARDISODispose();
Console.WriteLine("PARDISO ERROR: " + wrapper.PARDISOerror2string(error));
return(false);
}
//Console.Write("\nFactorization completed ...\n ");
}
/* -------------------------------------------------------------------- */
/* .. Back substitution and iterative refinement. */
/* -------------------------------------------------------------------- */
phase = 33;
iparm[7] = 1; /* Max numbers of iterative refinement steps. */
//m_foo.mkl_serv_mkl_set_num_threads(num_procs);
wrapper.PARDISO(pt, &maxfct, &mnum, &mtype, &phase,
&n, a, ia, ja, &idum, &nrhs,
iparm, &msglvl, b, x, &error, dparam);
//Console.WriteLine("33: IPARAM(22) = {0}, IPARAM(23) = {1}", iparm[21], iparm[22]);
if (error != 0)
{
PARDISODispose();
Console.WriteLine("PARDISO ERROR: " + wrapper.PARDISOerror2string(error));
return(false);
}
//Console.Write("\nSolve completed ... ");
}
}
}
}
#if DEBUG
Debug.Assert(ArrayTools.ListEquals <double>(aClone, m_PardisoMatrix.a), "PARDISO changed the matrix.");
Debug.Assert(ArrayTools.ListEquals <int>(iaClone, m_PardisoMatrix.ia), "PARDISO changed the matrix.");
Debug.Assert(ArrayTools.ListEquals <int>(jaClone, m_PardisoMatrix.ja), "PARDISO changed the matrix.");
#endif
}
m_PardInt.m_PardisoInitialized = true;
return(true);
}
}
/// <summary>
/// Creates a new grid, which is an adaptive refinement (cell by cell) of this grid.
/// </summary>
public GridCommons Adapt(IEnumerable <int> CellsToRefine, IEnumerable <int[]> CellsToCoarsen, out GridCorrelation Old2New)
{
using (new FuncTrace()) {
GridCommons oldGrid = this.m_Grid;
GridCommons newGrid = new GridCommons(oldGrid.RefElements, oldGrid.EdgeRefElements);
Old2New = new GridCorrelation();
int J = this.Cells.NoOfLocalUpdatedCells;
BitArray CellsToRefineBitmask = new BitArray(J);
BitArray CellsToCoarseBitmask = new BitArray(J);
BitArray AdaptNeighborsBitmask = new BitArray(J);
// templates for subdivision
// =========================
RefElement[] KrefS = oldGrid.RefElements; // all ref elements used
RefElement.SubdivisionTreeNode[] KrefS_SubDiv = new RefElement.SubdivisionTreeNode[KrefS.Length]; // subdivision tree for each ref element
RefElement.SubdivisionTreeNode[][] KrefS_SubdivLeaves = new RefElement.SubdivisionTreeNode[KrefS.Length][]; // actual subdivision elements
Tuple <int, int>[][,] KrefS_SubdivConnections = new Tuple <int, int> [KrefS.Length][, ]; // connections between elements; 1st idx: ref elem; 2nd idx: subdiv elm; 3rd idx: face of subdiv elm; content: [idx of subdiv elm,idx of face]
int[][][] KrefS_Faces2Subdiv = new int[KrefS.Length][][]; // mapping: [ref elm, face of ref elm] -> Subdivision elements which bound to this face.
Old2New.GeometricMapping = new AffineTrafo[KrefS.Length][];
//List<AffineTrafo> InterCellTrafos = new List<AffineTrafo>();
//int[][] RefinementIctIdx = new int[KrefS.Length][];
//int[][] CoarseningIctIdx = new int[KrefS.Length][];
for (int iKref = 0; iKref < KrefS.Length; iKref++)
{
RefElement Kref = KrefS[iKref];
KrefS_SubDiv[iKref] = Kref.GetSubdivisionTree(1);
KrefS_SubdivLeaves[iKref] = KrefS_SubDiv[0].GetLeaves();
Debug.Assert(ArrayTools.ListEquals(KrefS_SubdivLeaves[iKref], KrefS_SubDiv[iKref].Children[0].GetLevel(), (a, b) => object.ReferenceEquals(a, b)));
KrefS_Faces2Subdiv[iKref] = new int[Kref.NoOfFaces][];
KrefS_SubdivConnections[iKref] = new Tuple <int, int> [KrefS_SubdivLeaves[iKref].Length, KrefS[iKref].NoOfFaces];
for (int iSubdiv = 0; iSubdiv < KrefS_SubdivConnections[iKref].GetLength(0); iSubdiv++) // loop over subdivision elements
{
for (int iFace = 0; iFace < KrefS_SubdivConnections[iKref].GetLength(1); iFace++) // loop over faces of subdivision elements
{
var t = KrefS_SubdivLeaves[iKref][iSubdiv].GetNeighbor(iFace);
if (t.Item1 < 0)
{
// at the boundary of the subdivision
ArrayTools.AddToArray(iSubdiv, ref KrefS_Faces2Subdiv[iKref][t.Item2]);
}
KrefS_SubdivConnections[iKref][iSubdiv, iFace] = t;
}
}
//RefinementIctIdx[iKref] = new int[KrefS_SubdivLeaves[iKref].Length];
//CoarseningIctIdx[iKref] = new int[KrefS_SubdivLeaves[iKref].Length];
Old2New.GeometricMapping[iKref] = new AffineTrafo[KrefS_SubdivLeaves[iKref].Length];
for (int iSubDiv = 0; iSubDiv < KrefS_SubdivLeaves[iKref].Length; iSubDiv++)
{
Old2New.GeometricMapping[iKref][iSubDiv] = KrefS_SubdivLeaves[iKref][iSubDiv].TrafoFromRoot;
//InterCellTrafos.Add(KrefS_SubdivLeaves[iKref][iSubDiv].TrafoFromRoot);
//RefinementIctIdx[iKref][iSubDiv] = InterCellTrafos.Count - 1;
//InterCellTrafos.Add(KrefS_SubdivLeaves[iKref][iSubDiv].Trafo2Root);
//CoarseningIctIdx[iKref][iSubDiv] = InterCellTrafos.Count - 1;
}
}
Old2New.KrefS_SubdivLeaves = KrefS_SubdivLeaves;
// Check Input, set Bitmasks
// =========================
if (CellsToRefine != null)
{
foreach (int jCell in CellsToRefine)
{
if (CellsToRefineBitmask[jCell] == true)
{
throw new ArgumentException("Double entry.", "CellsToRefine");
}
CellsToRefineBitmask[jCell] = true;
int[] Neighs, dummy;
this.GetCellNeighbours(jCell, GetCellNeighbours_Mode.ViaEdges, out Neighs, out dummy);
foreach (int jNeigh in Neighs)
{
AdaptNeighborsBitmask[jNeigh] = true;
}
}
}
if (CellsToCoarsen != null)
{
foreach (int[] jCellS in CellsToCoarsen) // loop over all coarsening clusters...
// cluster of cells to coarsen
{
Cell[] CellS = jCellS.Select(j => this.Cells.GetCell(j)).ToArray();
int CoarseningClusterID = CellS[0].CoarseningClusterID;
int iKref = this.Cells.GetRefElementIndex(jCellS[0]);
int RefinementLevel = CellS[0].RefinementLevel;
if (jCellS.Length != KrefS_SubdivLeaves[iKref].Length)
{
throw new ArgumentException("Number of elements in coarsening cluster does not match refinement template for respective element type.");
}
if (RefinementLevel <= 0 || CoarseningClusterID <= 0)
{
throw new ArgumentException("Coarsening not available for respective cell.");
}
if (CellS.Where(cl => cl.ParentCell != null).Count() != 1)
{
throw new ArgumentException("Coarsening cluster seems wrong, or internal data may be corrupted.");
}
for (int z = 0; z < CellS.Length; z++)
{
int j = jCellS[z];
if (CellsToRefineBitmask[j] == true)
{
throw new ArgumentException("Cannot refine and coarsen the same cell.");
}
if (CellsToCoarseBitmask[j] == true)
{
throw new ArgumentException("Double entry.", "CellsToCoarsen");
}
CellsToCoarseBitmask[j] = true;
Cell Cj = this.Cells.GetCell(j);
//if(Cj.CoarseningPeers == null)
// throw new ArgumentException("Coarsening not available for respective cell.");
//if(Cj.CoarseningPeers.Length != jCellS.Length - 1)
// throw new ArgumentException("Coarsening cluster seems incomplete.");
//if(Cj.CoarseningPeers.Length != jCellS.Length - 1)
// throw new ArgumentException("Coarsening cluster seems incomplete.");
//foreach(long gid in jCellS) {
// if(CellS.Where(cl => cl.GlobalID == gid).Count() != 1)
// throw new ArgumentException("Coarsening cluster seems incomplete.");
//}
if (CoarseningClusterID != Cj.CoarseningClusterID)
{
throw new ArgumentException("Mismatch of 'CoarseningClusterID' within cluster.");
}
int[] Neighs, dummy;
this.GetCellNeighbours(j, GetCellNeighbours_Mode.ViaVertices, out Neighs, out dummy);
foreach (int jNeigh in Neighs)
{
if (Array.IndexOf(jCellS, jNeigh) < 0)
{
AdaptNeighborsBitmask[jNeigh] = true;
}
}
}
}
}
int InsertCounter = J;
// create new cells
// ================
Debug.Assert(this.MpiSize == 1, "still need to adjust the following lines.");
long GlobalIdCounter = oldGrid.NumberOfCells_l;
//int PtrNewCells = oldGrid.NoOfUpdateCells;
//newGrid.Cells = new Cell[NewNoOfCells];
List <Cell> newCells = new List <Cell>();
int newVertexCounter = oldGrid.Cells.Max(cl => cl.NodeIndices.Max()) + 1;
Cell[][] adaptedCells = new Cell[J][];
Old2New.OldGlobalId = this.CurrentGlobalIdPermutation.Values.CloneAs();
Old2New.MappingIndex = new int[J][];
Old2New.DestGlobalId = new long[J][];
// clone neighbors of refined/coarsened cells
// ------------------------------------------
for (int j = 0; j < J; j++)
{
Debug.Assert(Old2New.OldGlobalId[j] == this.Cells.GetCell(j).GlobalID);
Debug.Assert(Old2New.OldGlobalId[j] == oldGrid.Cells[j].GlobalID);
Debug.Assert(object.ReferenceEquals(this.Cells.GetCell(j), oldGrid.Cells[j]));
Debug.Assert((CellsToRefineBitmask[j] && CellsToCoarseBitmask[j]) == false, "Cannot refine and coarsen the same cell.");
if ((CellsToRefineBitmask[j] || CellsToCoarseBitmask[j]) == false)
{
if (AdaptNeighborsBitmask[j])
{
// neighbor information needs to be updated
var oldCell = oldGrid.Cells[j];
var newCell = oldCell.CloneAs(); // data
newCells.Add(newCell);
adaptedCells[j] = new Cell[] { newCell };
// remove out-dated neighborship info
if (newCell.CellFaceTags != null && newCell.CellFaceTags.Length > 0)
{
int[] oldNeighs = this.Cells.CellNeighbours[j];
foreach (int jNeigh in oldNeighs)
{
if (CellsToRefineBitmask[jNeigh] || CellsToCoarseBitmask[jNeigh])
{
// one of the neighbors has changed, so _potentially_ the cell face tags have to be updated
long gId_Neigh = this.Cells.GetGlobalID(jNeigh);
for (int i = 0; i < newCell.CellFaceTags.Length; i++)
{
if (newCell.CellFaceTags[i].NeighCell_GlobalID == gId_Neigh)
{
Debug.Assert(newCell.CellFaceTags[i].EdgeTag == 0 || newCell.CellFaceTags[i].EdgeTag >= GridCommons.FIRST_PERIODIC_BC_TAG);
ArrayTools.RemoveAt(ref newCell.CellFaceTags, i);
i--;
}
}
}
}
}
}
else
{
// cell and neighbors remain unchanged
newCells.Add(oldGrid.Cells[j]);
}
Debug.Assert(Old2New.MappingIndex[j] == null);
Debug.Assert(Old2New.DestGlobalId[j] == null);
Old2New.MappingIndex[j] = null;
Old2New.DestGlobalId[j] = new long[] { newCells[newCells.Count - 1].GlobalID };
}
else
{
Debug.Assert(CellsToRefineBitmask[j] || CellsToCoarseBitmask[j]);
}
}
// coarsening
// ----------
int bCoarsened = 0;
if (CellsToCoarsen != null)
{
foreach (int[] jCellS in CellsToCoarsen)
{
bCoarsened = 0xFFFF;
// cluster of cells to coarsen
Cell[] CellS = jCellS.Select(j => this.Cells.GetCell(j)).ToArray();
Debug.Assert(jCellS.Length == CellS.Length);
int RefinementLevel = CellS[0].RefinementLevel - 1;
if (RefinementLevel < 0)
{
throw new ArgumentException("Refinement level out of range - corrupted data structure.");
}
foreach (var cl in CellS)
{
if (cl.RefinementLevel != RefinementLevel + 1)
{
throw new ArgumentException("Refinement varies within refinement cluster - corrupted data structure.");
}
}
Cell Cell0 = CellS.Single(cl => cl.ParentCell != null);
Cell Mother = Cell0.ParentCell;
//Debug.Assert(CellS.Where(cl => cl.GlobalID == Mother.GlobalID).Count() == 1);
Debug.Assert(Mother.RefinementLevel == RefinementLevel);
Cell restoredCell = new Cell();
restoredCell.Type = Mother.Type;
Debug.Assert(Mother.Type == Cell0.Type);
restoredCell.NodeIndices = Mother.NodeIndices;
restoredCell.CoarseningClusterID = Mother.CoarseningClusterID;
restoredCell.ParentCell = Mother.ParentCell;
restoredCell.GlobalID = Cell0.GlobalID;
restoredCell.TransformationParams = Mother.TransformationParams;
restoredCell.RefinementLevel = RefinementLevel;
restoredCell.CoarseningClusterSize = Mother.CoarseningClusterSize;
restoredCell.CoarseningLeafIndex = Mother.CoarseningLeafIndex;
// boundary conditions by cell face tags
if (Mother.CellFaceTags != null)
{
restoredCell.CellFaceTags = Mother.CellFaceTags.Where(cftag => cftag.EdgeTag > 0 && cftag.EdgeTag < GridCommons.FIRST_PERIODIC_BC_TAG).ToArray();
}
for (int iSubDiv = 0; iSubDiv < jCellS.Length; iSubDiv++)
{
int j = jCellS[iSubDiv];
Cell Cj = CellS[iSubDiv];
Debug.Assert(adaptedCells[j] == null);
adaptedCells[j] = new[] { restoredCell };
Debug.Assert(Old2New.MappingIndex[j] == null);
Debug.Assert(Old2New.DestGlobalId[j] == null);
Old2New.MappingIndex[j] = new int[] { Cj.CoarseningLeafIndex };
Old2New.DestGlobalId[j] = new long[] { restoredCell.GlobalID };
}
newCells.Add(restoredCell);
}
}
// refinement
// ----------
if (CellsToRefine != null)
{
int NewCoarseningClusterId;
{
int[] locData = new int[] {
(this.m_Grid.Cells.Max(cl => cl.CoarseningClusterID)),
(CellsToRefine.Count() + 1)
};
int[] glbData = locData.MPIMax();
NewCoarseningClusterId = glbData[0] + 1 + glbData[1] * this.MpiRank;
}
foreach (int j in CellsToRefine)
{
var oldCell = oldGrid.Cells[j];
int iKref = this.Cells.GetRefElementIndex(j);
var Kref = KrefS[iKref];
var Leaves = KrefS_SubdivLeaves[iKref];
Tuple <int, int>[,] Connections = KrefS_SubdivConnections[iKref];
NodeSet RefNodes = Kref.GetInterpolationNodes(oldCell.Type);
Debug.Assert(adaptedCells[j] == null);
Cell[] refinedCells = new Cell[Leaves.Length];
adaptedCells[j] = refinedCells;
Debug.Assert(Old2New.MappingIndex[j] == null);
Old2New.MappingIndex[j] = new int[Leaves.Length];
for (int iSubDiv = 0; iSubDiv < Leaves.Length; iSubDiv++) // pass 1: create new cells
// create new cell
{
Cell newCell = new Cell();
newCell.Type = oldCell.Type;
if (iSubDiv == 0)
{
newCell.GlobalID = oldCell.GlobalID;
newCell.ParentCell = oldCell.CloneAs();
}
else
{
newCell.GlobalID = GlobalIdCounter;
GlobalIdCounter++;
}
newCell.RefinementLevel = oldCell.RefinementLevel + 1;
newCell.CoarseningClusterSize = Leaves.Length;
newCell.CoarseningClusterID = NewCoarseningClusterId;
newCell.CoarseningLeafIndex = iSubDiv;
refinedCells[iSubDiv] = newCell;
// Vertices
var RefNodesRoot = Leaves[iSubDiv].Trafo2Root.Transform(RefNodes);
newCell.TransformationParams = MultidimensionalArray.Create(RefNodes.Lengths);
this.TransformLocal2Global(RefNodesRoot, newCell.TransformationParams, j);
// node indices
newCell.NodeIndices = new int[Kref.NoOfVertices];
for (int i = 0; i < Kref.NoOfVertices; i++)
{
newCell.NodeIndices[i] = newVertexCounter;
newVertexCounter++;
}
// correlation
Old2New.MappingIndex[j][iSubDiv] = iSubDiv;
}
NewCoarseningClusterId++;
for (int iSubDiv = 0; iSubDiv < Leaves.Length; iSubDiv++) // pass 2: do other things
//// record information for (later) coarsening
//refinedCells[iSubDiv].CoarseningPeers = refinedCells
// .Where(cell => cell.GlobalID != refinedCells[iSubDiv].GlobalID)
// .Select(cell => cell.GlobalID)
// .ToArray();
// neighbors within
{
for (int iFace = 0; iFace < Kref.NoOfFaces; iFace++)
{
int iSubDiv_Neigh = Connections[iSubDiv, iFace].Item1;
if (iSubDiv_Neigh >= 0)
{
ArrayTools.AddToArray(new CellFaceTag()
{
ConformalNeighborship = true,
NeighCell_GlobalID = refinedCells[Connections[iSubDiv, iFace].Item1].GlobalID,
FaceIndex = iFace
}, ref refinedCells[iSubDiv].CellFaceTags);
}
}
}
newCells.AddRange(refinedCells);
Debug.Assert(Old2New.DestGlobalId[j] == null);
Old2New.DestGlobalId[j] = refinedCells.Select(Cl => Cl.GlobalID).ToArray();
}
}
newGrid.Cells = newCells.ToArray();
// fix neighborship
// ================
byte[,] Edge2Face = this.Edges.FaceIndices;
//int[][] Cells2Edges = this.Cells.Cells2Edges;
int[,] Edge2Cell = this.Edges.CellIndices;
//byte[] EdgeTags = this.Edges.EdgeTags;
MultidimensionalArray[] VerticesFor_KrefEdge = this.Edges.EdgeRefElements.Select(KrefEdge => KrefEdge.Vertices).ToArray();
int[] ONE_NULL = new int[] { 0 };
int NoOfEdges = this.Edges.Count;
Debug.Assert(Edge2Face.GetLength(0) == NoOfEdges);
Debug.Assert(Edge2Cell.GetLength(0) == NoOfEdges);
for (int iEdge = 0; iEdge < NoOfEdges; iEdge++)
{
int jCell1 = Edge2Cell[iEdge, 0];
int jCell2 = Edge2Cell[iEdge, 1];
if (jCell2 < 0)
{
continue;
}
Debug.Assert((CellsToRefineBitmask[jCell1] && CellsToCoarseBitmask[jCell1]) == false);
Debug.Assert((CellsToRefineBitmask[jCell2] && CellsToCoarseBitmask[jCell2]) == false);
bool C1changed = CellsToRefineBitmask[jCell1] || CellsToCoarseBitmask[jCell1];
bool C2changed = CellsToRefineBitmask[jCell2] || CellsToCoarseBitmask[jCell2];
if ((C1changed || C2changed) == false)
{
// edge between two un-changed cells -- this neighborship remains the same.
continue;
}
Cell[] adaptedCells1 = adaptedCells[jCell1];
Cell[] adaptedCells2 = adaptedCells[jCell2];
if (CellsToCoarseBitmask[jCell1] && CellsToCoarseBitmask[jCell2])
{
Debug.Assert(adaptedCells1.Length == 1);
Debug.Assert(adaptedCells2.Length == 1);
if (adaptedCells1[0].GlobalID == adaptedCells2[0].GlobalID)
{
// these two cells will be joint into one cell -> no new neighborship
Debug.Assert(ReferenceEquals(adaptedCells1[0], adaptedCells2[0]));
continue;
}
}
Debug.Assert(adaptedCells1 != null);
Debug.Assert(adaptedCells2 != null);
int iFace1 = Edge2Face[iEdge, 0];
int iFace2 = Edge2Face[iEdge, 1];
Debug.Assert((adaptedCells1.Length > 1) == (CellsToRefineBitmask[jCell1]));
Debug.Assert((adaptedCells2.Length > 1) == (CellsToRefineBitmask[jCell2]));
int iKref1 = this.Cells.GetRefElementIndex(jCell1);
int iKref2 = this.Cells.GetRefElementIndex(jCell2);
var Kref1 = this.Cells.GetRefElement(jCell1);
var Kref2 = this.Cells.GetRefElement(jCell2);
int[] idx1, idx2;
if (CellsToRefineBitmask[jCell1])
{
idx1 = KrefS_Faces2Subdiv[iKref1][iFace1];
}
else
{
Debug.Assert(adaptedCells1.Length == 1);
idx1 = ONE_NULL;
}
if (CellsToRefineBitmask[jCell2])
{
idx2 = KrefS_Faces2Subdiv[iKref2][iFace2];
}
else
{
Debug.Assert(adaptedCells2.Length == 1);
idx2 = ONE_NULL;
}
foreach (int i1 in idx1)
{
MultidimensionalArray VtxFace1;
if (CellsToRefineBitmask[jCell1])
{
VtxFace1 = KrefS_SubdivLeaves[iKref1][i1].GetFaceVertices(iFace1);
}
else
{
VtxFace1 = Kref1.GetFaceVertices(iFace1);
}
Cell Cl1 = adaptedCells1[i1];
foreach (int i2 in idx2)
{
Cell Cl2 = adaptedCells2[i2];
Debug.Assert(Cl1.GlobalID != Cl2.GlobalID);
int conCount1;
if (Cl1.CellFaceTags == null)
{
conCount1 = 0;
}
else
{
conCount1 = Cl1.CellFaceTags.Where(cfTag => cfTag.NeighCell_GlobalID == Cl2.GlobalID).Count();
}
Debug.Assert(conCount1 <= 1);
#if DEBUG
int conCount2;
if (Cl2.CellFaceTags == null)
{
conCount2 = 0;
}
else
{
conCount2 = Cl2.CellFaceTags.Where(cfTag => cfTag.NeighCell_GlobalID == Cl1.GlobalID).Count();
}
Debug.Assert(conCount1 == conCount2);
#endif
if (conCount1 > 0)
{
continue;
}
MultidimensionalArray VtxFace2;
{
MultidimensionalArray VtxFace2_L;
if (CellsToRefineBitmask[jCell2])
{
VtxFace2_L = KrefS_SubdivLeaves[iKref2][i2].GetFaceVertices(iFace2);
}
else
{
VtxFace2_L = Kref2.GetFaceVertices(iFace2);
}
MultidimensionalArray VtxFace2_G = MultidimensionalArray.Create(VtxFace2_L.GetLength(0), VtxFace2_L.GetLength(1));
VtxFace2 = MultidimensionalArray.Create(VtxFace2_L.GetLength(0), VtxFace2_L.GetLength(1));
this.TransformLocal2Global(VtxFace2_L, VtxFace2_G, jCell2);
bool[] Converged = new bool[VtxFace2_L.NoOfRows];
this.TransformGlobal2Local(VtxFace2_G, VtxFace2, jCell1, Converged);
if (Converged.Any(t => t == false))
{
throw new ArithmeticException("Newton divergence");
}
}
bool bIntersect = GridData.EdgeData.FaceIntersect(VtxFace1, VtxFace2,
Kref1.GetFaceTrafo(iFace1), Kref1.GetInverseFaceTrafo(iFace1),
VerticesFor_KrefEdge,
out bool conformal1, out bool conformal2, out AffineTrafo newTrafo, out int Edg_idx);
if (bIntersect)
{
ArrayTools.AddToArray(new CellFaceTag()
{
ConformalNeighborship = false,
NeighCell_GlobalID = Cl2.GlobalID,
FaceIndex = iFace1
}, ref Cl1.CellFaceTags);
ArrayTools.AddToArray(new CellFaceTag()
{
ConformalNeighborship = false,
NeighCell_GlobalID = Cl1.GlobalID,
FaceIndex = iFace2
}, ref Cl2.CellFaceTags);
}
}
}
}
// finalize
// ========
int bCoarsenedGlobal = bCoarsened.MPIMax();
if (bCoarsenedGlobal > 0)
{
#if DEBUG
if (this.MpiSize == 1)
{
List <int> allgids = new List <int>();
foreach (var cl in newGrid.Cells)
{
allgids.Add((int)(cl.GlobalID));
}
bool[] markers = new bool[allgids.Max() + 1];
for (int i = 0; i < allgids.Count; i++)
{
long gid = allgids[i];
Debug.Assert(markers[gid] == false, "Some GlobalID is used twice.");
markers[gid] = true;
}
foreach (var cl in newGrid.Cells)
{
if (cl.CellFaceTags != null)
{
for (int i = 0; i < cl.CellFaceTags.Length; i++)
{
long ngid = cl.CellFaceTags[i].NeighCell_GlobalID;
if (ngid >= 0)
{
Debug.Assert(markers[ngid] == true);
}
}
}
}
}
#endif
List <long> old2NewGid = new List <long>();
Debug.Assert(Old2New.DestGlobalId.Length == J);
for (int j = 0; j < J; j++)
{
old2NewGid.AddRange(Old2New.DestGlobalId[j]);
}
newGrid.CompressGlobalID(old2NewGid);
int c2 = 0;
for (int j = 0; j < J; j++)
{
long[] o2nj = Old2New.DestGlobalId[j];
int K = o2nj.Length;
for (int k = 0; k < K; k++)
{
o2nj[k] = old2NewGid[c2];
c2++;
}
}
Debug.Assert(c2 == old2NewGid.Count);
}
return(newGrid);
}
}
BlockPartitioning GetBlocking(bool bConstantFrame)
{
int J = GridDat.iLogicalCells.NoOfLocalUpdatedCells;
Debug.Assert(J == GridDat.CellPartitioning.LocalLength);
Basis[] basisS = this.m_BasisS;
int NoOfBasisS = basisS.Length;
int[] BlockType = new int[J];
List <int[]> _SubblkLen = new List <int[]>();
List <int[]> _Subblk_i0 = new List <int[]>();
int[] SubblkLen_j = null;
int[] Subblk_i0_j = null;
for (int j = 0; j < J; j++)
{
if (SubblkLen_j == null)
{
SubblkLen_j = new int[NoOfBasisS];
}
if (Subblk_i0_j == null)
{
Subblk_i0_j = new int[NoOfBasisS];
}
int i0 = 0;
for (int iBs = 0; iBs < NoOfBasisS; iBs++)
{
Debug.Assert(i0 == m_j0CoordinateIndex[iBs]);
Subblk_i0_j[iBs] = m_j0CoordinateIndex[iBs];
SubblkLen_j[iBs] = basisS[iBs].GetLength(j);
i0 += basisS[iBs].MaximalLength;
}
Debug.Assert(_SubblkLen.Count == _Subblk_i0.Count);
int K = _SubblkLen.Count;
int blockType;
bool bFound = false;
for (blockType = 0; blockType < K; blockType++) // loop over all block types found so far
{
int[] len_bT = _SubblkLen[blockType];
int[] _i0_bT = _Subblk_i0[blockType];
Debug.Assert(len_bT.Length == NoOfBasisS);
Debug.Assert(_i0_bT.Length == NoOfBasisS);
Debug.Assert(ArrayTools.ListEquals <int>(_i0_bT, m_j0CoordinateIndex.GetSubVector(0, NoOfBasisS)));
bFound = true;
for (int i = 0; i < NoOfBasisS; i++)
{
if (len_bT[i] != SubblkLen_j[i])
{
bFound = false;
break;
}
if (_i0_bT[i] != Subblk_i0_j[i])
{
bFound = false;
break;
}
}
if (bFound)
{
break;
}
}
if (bFound == false)
{
Debug.Assert(blockType == _SubblkLen.Count);
Debug.Assert(blockType == _Subblk_i0.Count);
_Subblk_i0.Add(Subblk_i0_j.CloneAs());
_SubblkLen.Add(SubblkLen_j.CloneAs());
}
BlockType[j] = blockType;
}
return(new BlockPartitioning(
this.LocalLength,
bConstantFrame ? this.MaxTotalNoOfCoordinatesPerCell : -1,
_Subblk_i0.ToArray(), _SubblkLen.ToArray(),
BlockType,
this.MPI_Comm));
}
/// <summary>
/// computes the injector for multigrid level 1
/// </summary>
/// <seealso cref="BuildInjector_Lv2andup"/>
private static MultidimensionalArray[] BuildInjector_Lv1(
Basis maxDgBasis, int Np,
MultidimensionalArray InjectorsBase, bool[] InjectorsBaseReady,
int Jagg, int[][] Ag2Pt, int[][] C2F)
{
using (new FuncTrace()) {
MultidimensionalArray ortho = MultidimensionalArray.Create(Np, Np);
MultidimensionalArray[] Injectors_iLevel = new MultidimensionalArray[Jagg];
#if DEBUG
{
int Jbase = InjectorsBase.GetLength(0);
if (InjectorsBase.GetLength(1) != InjectorsBase.GetLength(2))
{
throw new ArgumentException();
}
int N = InjectorsBase.GetLength(1);
var check = MultidimensionalArray.Create(N, N);
for (int j = 0; j < Jbase; j++)
{
check.Clear();
check.AccEye(1.0);
check.Acc(-1.0, InjectorsBase.ExtractSubArrayShallow(j, -1, -1));
if (check.InfNorm() != 0.0)
{
throw new ArgumentException();
}
}
}
#endif
for (int j = 0; j < Jagg; j++) // loop over aggregate cells
{
Debug.Assert(ArrayTools.ListEquals(Ag2Pt[j], C2F[j]));
int[] compCell = Ag2Pt[j];
int I = compCell.Length;
Injectors_iLevel[j] = MultidimensionalArray.Create(I, Np, Np);
if (I > 1)
{
// compute extrapolation
int[,] CellPairs = new int[I - 1, 2];
for (int i = 0; i < I - 1; i++)
{
CellPairs[i, 0] = compCell[0];
CellPairs[i, 1] = compCell[i + 1];
}
var ExpolMtx = MultidimensionalArray.Create(I, Np, Np);
maxDgBasis.GetExtrapolationMatrices(CellPairs, ExpolMtx.ExtractSubArrayShallow(new int[] { 1, 0, 0 }, new int[] { I - 1, Np - 1, Np - 1 }));
for (int n = 0; n < Np; n++)
{
ExpolMtx[0, n, n] = 1.0;
}
// Compute intermediate mass matrix
var MMtemp = MultidimensionalArray.Create(Np, Np);
MMtemp.Multiply(1.0, ExpolMtx, ExpolMtx, 0.0, "nm", "iln", "ilm");
// orthonormalize
MMtemp.SymmetricLDLInversion(ortho, null); // ortho is output, will be overwritten
Injectors_iLevel[j].Multiply(1.0, ExpolMtx, ortho, 0.0, "inm", "ink", "km");
}
else
{
Injectors_iLevel[j].ExtractSubArrayShallow(0, -1, -1).AccEye(1.0);
}
// base level injector
var injBase = InjectorsBase.ExtractSubArrayShallow(compCell[0], -1, -1);
injBase.Set(Injectors_iLevel[j].ExtractSubArrayShallow(0, -1, -1));
Debug.Assert(InjectorsBaseReady[compCell[0]]);
for (int i = 1; i < I; i++)
{
InjectorsBaseReady[compCell[i]] = false;
}
}
return(Injectors_iLevel);
}
}
//
//
/// <summary>
/// applies the agglomeration on a general matrix
/// </summary>
/// <param name="Matrix">the matrix that should be manipulated.</param>
/// <param name="Rhs">the right-hand-side that should be manipulated</param>
/// <param name="ColMap"></param>
/// <param name="ColMapAggSw">Turns column agglomeration on/off fore each variable individually; default == null is on. </param>
/// <param name="RowMap"></param>
/// <param name="RowMapAggSw">The same shit as for <paramref name="ColMapAggSw"/>, just for rows.</param>
public void ManipulateMatrixAndRHS<M, T>(M Matrix, T Rhs, UnsetteledCoordinateMapping RowMap, UnsetteledCoordinateMapping ColMap, bool[] RowMapAggSw = null, bool[] ColMapAggSw = null)
where M : IMutableMatrixEx
where T : IList<double>
{
MPICollectiveWatchDog.Watch();
//var mtxS = GetFrameMatrices(Matrix, RowMap, ColMap);
if (Matrix == null && Rhs == null)
// nothing to do
return;
if (TotalNumberOfAgglomerations <= 0)
// nothing to do
return;
if (RowMapAggSw != null)
throw new NotImplementedException();
// generate agglomeration sparse matrices
// ======================================
int RequireRight;
if (Matrix == null) {
// we don't need multiplication-from-the-right at all
RequireRight = 0;
} else {
if (RowMap.EqualsUnsetteled(ColMap) && ArrayTools.ListEquals(ColMapAggSw, RowMapAggSw)) {
// we can use the same matrix for right and left multiplication
RequireRight = 1;
} else {
// separate matrix for the multiplication-from-the-right is required
RequireRight = 2;
}
}
BlockMsrMatrix LeftMul = null, RightMul = null;
{
foreach (var kv in DictAgglomeration) {
var Species = kv.Key;
var m_Agglomerator = kv.Value;
if (m_Agglomerator != null) {
CellMask spcMask = this.Tracker.Regions.GetSpeciesMask(Species);
MiniMapping rowMini = new MiniMapping(RowMap, Species, this.Tracker.Regions);
BlockMsrMatrix LeftMul_Species = m_Agglomerator.GetRowManipulationMatrix(RowMap, rowMini.MaxDeg, rowMini.NoOfVars, rowMini.i0Func, rowMini.NFunc, false, spcMask);
if (LeftMul == null) {
LeftMul = LeftMul_Species;
} else {
LeftMul.Acc(1.0, LeftMul_Species);
}
if (!object.ReferenceEquals(LeftMul, RightMul) && RightMul != null) {
MiniMapping colMini = new MiniMapping(ColMap, Species, this.Tracker.Regions);
BlockMsrMatrix RightMul_Species = m_Agglomerator.GetRowManipulationMatrix(ColMap, colMini.MaxDeg, colMini.NoOfVars, colMini.i0Func, colMini.NFunc, false, spcMask);
if (RightMul == null) {
RightMul = RightMul_Species;
} else {
RightMul.Acc(1.0, RightMul_Species);
}
} else if (RequireRight == 1) {
RightMul = LeftMul;
} else {
RightMul = null;
}
}
}
}
// apply the agglomeration to the matrix
// =====================================
if (Matrix != null) {
BlockMsrMatrix RightMulTr = RightMul.Transpose();
BlockMsrMatrix _Matrix;
if (Matrix is BlockMsrMatrix) {
_Matrix = (BlockMsrMatrix)((object)Matrix);
} else {
_Matrix = Matrix.ToBlockMsrMatrix(RowMap, ColMap);
}
var AggMatrix = BlockMsrMatrix.Multiply(LeftMul, BlockMsrMatrix.Multiply(_Matrix, RightMulTr));
if (object.ReferenceEquals(_Matrix, Matrix)) {
_Matrix.Clear();
_Matrix.Acc(1.0, AggMatrix);
} else {
Matrix.Acc(-1.0, _Matrix); // das ist so
Matrix.Acc(1.0, AggMatrix); // meagaschlecht !!!!!!
}
}
// apply the agglomeration to the Rhs
// ==================================
if (Rhs != null) {
double[] tmp = Rhs.ToArray();
if (object.ReferenceEquals(tmp, Rhs))
throw new ApplicationException("Flache kopie sollte eigentlich ausgeschlossen sein!?");
LeftMul.SpMV(1.0, tmp, 0.0, Rhs);
}
}
/// <summary>
/// only executed on proc 0
/// </summary>
bool PARDISOInitAndSolve(IMutableMatrixEx Mtx, double[] _x, double[] _b)
{
using (var tr = new FuncTrace()) {
//InitAndSolve.Start();
if (m_PardisoMatrix == null)
{
m_PardisoMatrix = new Matrix(Mtx, this.UseDoublePrecision);
}
int rank;
csMPI.Raw.Comm_Rank(MpiComm, out rank);
if (rank == 0)
{
#if DEBUG
long aSize = m_PardisoMatrix.ja.LongLength * (UseDoublePrecision ? sizeof(double) : sizeof(float));
//double[] a_DClone = null;
//float[] a_SClone = null;
//if(UseDoublePrecision)
// a_DClone = m_PardisoMatrix.a_D.CloneAs();
//else
// a_SClone = m_PardisoMatrix.a_S.CloneAs();
IntPtr aClone = Marshal.AllocHGlobal((IntPtr)aSize);
unsafe
{
int *psrc = (int *)m_PardisoMatrix.aPtr;
int *pdst = (int *)aClone;
for (long l = 0; l < aSize; l += sizeof(int))
{
*pdst = *psrc;
pdst++;
psrc++;
}
}
int[] iaClone = m_PardisoMatrix.ia.CloneAs();
int[] jaClone = m_PardisoMatrix.ja.CloneAs();
#endif
unsafe
{
fixed(double *px = _x, pb = _b, dparam = m_PardInt.m_dparam)
{
fixed(int *ia = m_PardisoMatrix.ia, ja = m_PardisoMatrix.ja, iparm = m_PardInt.m_parm, __pt = m_PardInt.m_pt)
{
int n = m_PardisoMatrix.n;
//int nnz = ia[n];
int mtype = GetMType();
int nrhs = 1; /* Number of right hand sides. */
/* Internal solver memory pointer pt, */
/* 32-bit: int pt[64]; 64-bit: long int pt[64] */
/* or void *pt[64] should be OK on both architectures */
void *pt = (void *)__pt;
/* Pardiso control parameters. */
int maxfct = m_PardInt.maxfct, mnum = m_PardInt.mnum, msglvl = m_PardInt.msglvl;
int phase, error;
/* Auxiliary variables. */
//int i;
double ddum; /* Double dummy */
int idum; /* Integer dummy. */
void * a = (void *)(m_PardisoMatrix.aPtr);
double *b, x;
if (UseDoublePrecision)
{
b = pb;
x = px;
}
else
{
b = (double *)Marshal.AllocHGlobal(n * sizeof(float));
x = (double *)Marshal.AllocHGlobal(n * sizeof(float));
float *bS = (float *)b;
float *xS = (float *)x;
for (int i = 0; i < n; i++)
{
bS[i] = (float)(pb[i]);
xS[i] = (float)(px[i]);
}
//SingleCalls.Start();
}
//Inner.Start();
if (!m_PardInt.m_PardisoInitialized)
{
/* -------------------------------------------------------------------- */
/* .. Setup Pardiso control parameters und initialize the solvers */
/* internal adress pointers. This is only necessary for the FIRST */
/* call of the PARDISO solver. */
/* ---------------------------------------------------------------------*/
if (this.Version == PARDISO.Version.MKL)
{
// Do nothing NUM_THREADS is set by environment variables in MKLPardiso
// iparm[2] stays set to zero
// If you want to control this manually, do something like this:
//System.Environment.SetEnvironmentVariable("OMP_NUM_THREADS", "4");
//System.Environment.SetEnvironmentVariable("MKL_NUM_THREADS", "4");
//If this is true: MKL determines the number of OMP threads automatically, based on proc information
//System.Environment.SetEnvironmentVariable("MKL_DYNAMIC", "false");
}
else if (this.Version == PARDISO.Version.v5)
{
// Get Value for IPARM(3) from the Environment Variable
int NumOfProcs = Convert.ToInt32(System.Environment.GetEnvironmentVariable("OMP_NUM_THREADS"));
iparm[2] = NumOfProcs;
#if DEBUG
Console.WriteLine("IPARM(3) - NumberOfProcessors set to {0}", iparm[2]);
#endif
}
using (new BlockTrace("PARDISOINIT", tr))
{
wrapper.PARDISOINIT(pt, &mtype, iparm, dparam);
}
//Console.WriteLine("init: IPARAM(22) = {0}, IPARAM(23) = {1}", iparm[21], iparm[22]);
iparm[27] = this.UseDoublePrecision ? 0 : 1; // set single or double precision
maxfct = 1; /* Maximum number of numerical factorizations. */
mnum = 1; /* Which factorization to use. */
msglvl = 0; /* Print statistical information */
error = 0; /* Initialize error flag */
/* -------------------------------------------------------------------- */
/* .. Reordering and Symbolic Factorization. This step also allocates */
/* all memory that is necessary for the factorization. */
/* -------------------------------------------------------------------- */
using (new BlockTrace("PARDISO_phase11", tr))
{
phase = 11;
iparm[59] = 0; // in-core (1 == out-of-core)
//Console.Write("calling pardiso, phase 11... ");
Phase_11.Start();
wrapper.PARDISO(pt, &maxfct, &mnum, &mtype, &phase,
&n, a, ia, ja, &idum, &nrhs,
iparm, &msglvl, &ddum, &ddum, &error, dparam);
Phase_11.Stop();
//Console.WriteLine("11: IPARAM(22) = {0}, IPARAM(23) = {1}", iparm[21], iparm[22]);
}
if (error != 0)
{
PARDISODispose();
Console.Error.WriteLine("PARDISO ERROR: " + wrapper.PARDISOerror2string(error));
return(false);
}
//Console.Write("\nReordering completed ... ");
//Console.Write("\nNumber of nonzeros in factors = %d", iparm[17]);
//Console.Write("\nNumber of factorization MFLOPS = %d", iparm[18]);
/* -------------------------------------------------------------------- */
/* .. Numerical factorization. */
/* -------------------------------------------------------------------- */
using (new BlockTrace("PARDISO_phase22", tr))
{
phase = 22;
Phase_22.Start();
wrapper.PARDISO(pt, &maxfct, &mnum, &mtype, &phase,
&n, a, ia, ja, &idum, &nrhs,
iparm, &msglvl, &ddum, &ddum, &error, dparam);
Phase_22.Stop();
}
if (error != 0)
{
// some error occured: release mem, dispose objects...
PARDISODispose();
Console.Error.WriteLine("PARDISO ERROR: " + wrapper.PARDISOerror2string(error));
//InitAndSolve.Stop();
return(false);
}
}
/* -------------------------------------------------------------------- */
/* .. Back substitution and iterative refinement. */
/* -------------------------------------------------------------------- */
phase = 33;
iparm[7] = 0; /* Max numbers of iterative refinement steps, 0 == auto */
//m_foo.mkl_serv_mkl_set_num_threads(num_procs);
using (new BlockTrace("PARDISO_phase33", tr))
{
Phase_33.Start();
wrapper.PARDISO(pt, &maxfct, &mnum, &mtype, &phase,
&n, a, ia, ja, &idum, &nrhs,
iparm, &msglvl, b, x, &error, dparam);
Phase_33.Stop();
}
if (error != 0)
{
// some error occurred: release mem, dispose objects...
PARDISODispose();
Console.Error.WriteLine("PARDISO ERROR: " + wrapper.PARDISOerror2string(error));
//InitAndSolve.Stop();
return(false);
}
//Inner.Stop();
if (UseDoublePrecision)
{
}
else
{
//SingleCalls.Stop();
float *bS = (float *)b;
float *xS = (float *)x;
for (int i = 0; i < n; i++)
{
pb[i] = (bS[i]);
px[i] = (xS[i]);
}
Marshal.FreeHGlobal((IntPtr)b);
Marshal.FreeHGlobal((IntPtr)x);
}
}
}
}
#if DEBUG
//if(UseDoublePrecision)
// Debug.Assert(ArrayTools.ListEquals<double>(a_DClone, m_PardisoMatrix.a_D), "PARDISO changed the matrix.");
//else
// Debug.Assert(ArrayTools.ListEquals<float>(a_SClone, m_PardisoMatrix.a_S), "PARDISO changed the matrix.");
unsafe
{
int *psrc = (int *)m_PardisoMatrix.aPtr;
int *pdst = (int *)aClone;
for (long l = 0; l < aSize; l += sizeof(int))
{
Debug.Assert(*pdst == *psrc, "PARDISO changed the matrix.");
pdst++;
psrc++;
}
}
Debug.Assert(ArrayTools.ListEquals <int>(iaClone, m_PardisoMatrix.ia), "PARDISO changed the matrix.");
Debug.Assert(ArrayTools.ListEquals <int>(jaClone, m_PardisoMatrix.ja), "PARDISO changed the matrix.");
#endif
}
m_PardInt.m_PardisoInitialized = true;
//InitAndSolve.Stop();
return(true);
}
}
/// <summary>
/// See <see cref="T:int[].Equals"/>
/// </summary>
/// <param name="other"></param>
/// <returns></returns>
public bool Equals(TimestepNumber other)
{
return(ArrayTools.ListEquals(numbers, other.numbers));
}
/// <summary>
/// Saves a time-step to the database's persistent memory.
/// </summary>
/// <param name="_tsi">Contains Id etc.</param>
public void SaveTimestep(TimestepInfo _tsi)
{
using (var tr = new FuncTrace())
{
if (!(_tsi.ID.Equals(Guid.Empty) && _tsi.StorageID.Equals(Guid.Empty)))
{
throw new ArgumentException("Timestep is already saved in database");
}
var fields = _tsi.Fields.ToArray();
var GridDat = fields[0].GridDat;
{
List <DGField> FieldsFlatten = new List <DGField>();
TimestepInfo.FlattenHierarchy(FieldsFlatten, fields);
foreach (var f in FieldsFlatten)
{
if (!object.ReferenceEquals(f.GridDat, GridDat))
{
throw new ArgumentException("mismatch in GridData object.");
}
if (!fields.Contains(f, (a, b) => object.ReferenceEquals(a, b)))
{
// here, we ensure that the 'fields' -- list is complete, i.e.
// that the flatten hierarchy contains no field which is not already a memeber of 'fields'.
// The purpose is e.g. to prevent saving an XDG field without the required level-set field.
throw new ArgumentException(
"Unable to save timestep: field '" + f.Identification
+ "', which is required by at least one of the"
+ " given fields, must also be contained in the"
+ " given list of fields.",
"_tsi");
}
}
}
// build vector
// ============
int J = GridDat.iLogicalCells.NoOfLocalUpdatedCells;
var vec = new CellFieldDataSet[J];
var _fields = fields.ToArray();
int NF = _fields.Length;
var Permutation = GridDat.CurrentGlobalIdPermutation.Values;
for (int j = 0; j < J; j++)
{ // loop over cells
vec[j] = new CellFieldDataSet();
vec[j].GlobalID = Permutation[j];
//vec[j].DGCoordinateData = new CellFieldDataSet.CellFieldData[NF];
for (int idxF = 0; idxF < NF; idxF++)
{ // loop over fields
var field = _fields[idxF];
int N = field.Basis.GetLength(j);
double[] Coords = new double[N];
for (int n = 0; n < N; n++)
{
Coords[n] = field.Coordinates[j, n];
}
//vec[j].DGCoordinateData[idxF] = new CellFieldDataSet.CellFieldData() {
// Data = Coords
//};
vec[j].AppendDGCoordinates(Coords);
Debug.Assert(ArrayTools.ListEquals(Coords, vec[j].GetDGCoordinates(idxF)));
}
}
// Save dg coordinates
// ===================
Guid VectorGuid = Driver.SaveVector(vec);
_tsi.StorageID = VectorGuid;
// Save state object
// =================
_tsi.ID = Guid.NewGuid().MPIBroadcast(0);
Exception e = null;
if (MyRank == 0)
{
try
{
//tsi = new TimestepInfo(physTime, currentSession, TimestepNo, fields, VectorGuid);
using (var s = fsDriver.GetTimestepStream(true, _tsi.ID))
{
Driver.Serialize(s, _tsi, typeof(TimestepInfo));
s.Close();
}
}
catch (Exception ee)
{
e = ee;
Console.Error.WriteLine(ee.GetType().Name + " on rank " + MyRank + " saving time-step " + _tsi.TimeStepNumber + ": " + ee.Message);
Console.Error.WriteLine(ee.StackTrace);
}
}
e.ExceptionBcast();
// log session
// ===========
SessionInfo currentSession = (SessionInfo)(_tsi.Session); // hack
if (MyRank == 0)
{
try
{
currentSession.LogTimeStep(_tsi.ID);
}
catch (Exception ee)
{
e = ee;
Console.Error.WriteLine(ee.GetType().Name + " on rank " + MyRank + " saving time-step " + _tsi.TimeStepNumber + ": " + ee.Message);
Console.Error.WriteLine(ee.StackTrace);
}
}
e.ExceptionBcast();
_tsi.Database = currentSession.Database;
}
}