void PropagateByNeighborsSimple(Neighbor[][] nbList, float[][] a)
{
float avAff = 0.0f;
MT.ForEach(nbList, nbs => {
float s = nbs.Sum(nb => nb.distance * nb.distance);
lock (this)
avAff += s;
});
avAff = (float)Math.Sqrt(avAff / (nbList.Length * nbList[0].Length));
int rows = a.Length;
int columns = a[0].Length;
MT.Loop(0, rows, row => {
float[] newAff = new float[columns];
float[] aR = a[row];
for (int col = 0; col < columns; col++)
{
float aff = aR[col];
foreach (var nb in nbList[col])
{
if (nb.distance >= avAff)
{
break;
}
aff += aR[nb.index];
}
newAff[col] = aff;
}
a[row] = newAff;
});
}
public float[][] AffinityToDistance(float[][] P, float[][] Q)
{
// Symmetrize P, Q to P
const float eps = 2.22e-16f;
int rows = P.Length;
int columns = Q.Length;
MT.Loop(0, rows, row => {
for (int col = 0; col < columns; col++)
{
P[row][col] = P[row][col] + Q[col][row];
}
});
// Linearly map affinity to distance in to the range [0, 1.0].
var mmm = MinMaxMean(P);
float range = Math.Max(eps, mmm.Item2 - mmm.Item1);
float maxValue = mmm.Item2;
MT.Loop(0, rows, row => {
for (int col = 0; col < columns; col++)
{
P[row][col] = (maxValue - P[row][col]) / range;
}
});
return(P);
}
static void WriteMarix(DxShader.GpuDevice gpu, GBuf buffer, float[][] matrix, int col0, int col1, float[] uploadBuf)
{
int rows = matrix.Length;
Array.Clear(uploadBuf, 0, uploadBuf.Length);
MT.Loop(col0, col1, col => {
int offset = (col - col0) * rows;
for (int row = 0; row < rows; row++)
{
uploadBuf[offset + row] = matrix[row][col];
}
});
gpu.WriteArray(uploadBuf, buffer);
}
// Find the kNN neighbors of each node based on distance matrix cDist[].
Neighbor[][] FindNeighbors(float[][] cDist, int maxKnn)
{
int columns = cDist.Length;
int kNN = Math.Min(columns - 1, maxKnn);
Neighbor[][] nbList = new Neighbor[columns][]; // each row stores the kNN closet columns of a column.
MT.Loop(0, columns, col => {
// find the kNN nearest columns/neighbors of col-the column.
Neighbor[] nbs = nbList[col] = new Neighbor[kNN];
int kNN2 = 0; // The actually number of neighbors in nbs[]
for (int c = 0; c < columns; c++)
{
if (c == col)
{
continue;
}
float dist = (c < col) ? cDist[col][c] : cDist[c][col];
if (dist <= 0)
{
continue;
}
int i = 0;
for (i = kNN2 - 1; i >= 0; i--)
{
if (nbs[i].distance <= dist)
{
break;
}
}
// at here we have nbs[i].distance <= dist < nbs[i+1].distance
kNN2 = Math.Min(kNN, kNN2 + 1);
i++;
// Copy nbs[i:*] to nbs[i+1:*]
int len = kNN2 - i - 1;
if (len > 0)
{
Array.Copy(nbs, i, nbs, i + 1, len);
}
if (i < kNN2)
{
nbs[i].distance = dist;
nbs[i].index = c;
}
}
});
return(nbList);
}
static float[][] Transpose(float[][] matrix)
{
int rows = matrix.Length;
int columns = matrix[0].Length;
float[][] m = new float[columns][];
MT.Loop(0, columns, row => {
m[row] = new float[rows];
for (int col = 0; col < rows; col++)
{
m[row][col] = matrix[col][row];
}
});
return(m);
}
void PreCalculate() {
if (dataset == null)
return;
// Extract the relevant data table.
var bs = dataset.BodyList;
INumberTable nt = dataset.GetNumberTableView();
toIdx = Enumerable.Range(0, bs.Count).Where(i => !bs[i].Disabled).ToArray(); // Indexes of enabled bodies.
int N = nt.Rows - pcaSamples;
int[] enabledRows = toIdx.Where(i => i < N).ToArray();
if (enabledRows.Length == 0)
throw new TException("No data available!");
P = new float[enabledRows.Length][];
MT.Loop(0, P.Length, row => {
float[] R = P[row] = new float[nt.Columns];
double[] dsR = nt.Matrix[enabledRows[row]] as double[];
for (int col = 0; col < nt.Columns; col++)
R[col] = (float)dsR[col];
});
// Reverse toIdx;
int[] rIdx = Enumerable.Repeat(-1, bs.Count).ToArray();
for (int i = 0; i < toIdx.Length; i++) rIdx[toIdx[i]] = i;
toIdx = rIdx;
using (var gpu = new VisuMap.DxShader.GpuDevice())
dtP = DualAffinity.DotProduct(gpu, P, false);
float[] singValues = new float[pcaMax];
float[][] PC = FastPca.DoFastReduction(P, pcaMax, singValues, true);
P = VisuMap.MathUtil.NewMatrix<float>(PC.Length, pcaSamples); // P now links data points with the injected points on the main PCA axis.
float span = 4.0f * singValues[0];
stepSize = span / (pcaSamples - 1);
float x0 = - 0.5f * span;
MT.ForEach(PC, (R, row) => {
double yy = R.Skip(1).Sum(v => v * v);
for (int col = 0; col < pcaSamples; col++) {
double x = R[0] - (x0 + col * stepSize);
P[row][col] = (float)Math.Sqrt(x * x + yy);
}
});
}
void PropagateByNeighbors(Neighbor[][] nbList, float[][] a)
{
int kNN = nbList[0].Length;
int rows = a.Length;
int columns = a[0].Length;
// Calculate the average distance to kNN neighbors.
float avAff = 0.0f;
MT.ForEach(nbList, nbs => {
float s = nbs.Sum(nb => nb.distance);
lock (this)
avAff += s;
});
avAff /= nbList.Length * kNN;
// Prepare the exponentially descreasing coefficent to propgate neighboring affinity.
MT.ForEach(nbList, nbs => {
for (int k = 0; k < kNN; k++)
{
double e = nbs[k].distance / avAff;
nbs[k].distance = (float)Math.Exp(-0.5 * e * e);
}
});
// Propagate affinity from neibhoring columns.
MT.Loop(0, rows, row => {
float[] newAff = new float[columns];
float[] aR = a[row];
for (int col = 0; col < columns; col++)
{
var nbs = nbList[col];
float aff = aR[col];
foreach (var nb in nbs)
{
aff += aR[nb.index] * nb.distance;
}
newAff[col] = aff;
}
a[row] = newAff;
});
}
public List <string> Expression(List <string> selectedIds, double threshold)
{
const short maxExprIndex = 13; // number of expressed levels.
const short indexShift = 38;
double[][] M = (double[][])(NumTable.Matrix);
int cellNr = NumTable.Rows;
int geneNr = NumTable.Columns;
IList <int> selectedCells = NumTable.IndexOfRows(selectedIds);
IList <int> selectedGenes = NumTable.IndexOfColumns(selectedIds);
List <string> expressedId = new List <string>();
if (threshold == 0) // mark the genes/cells which have higher expression than the average of selected genes/cells.
{
if (GeneToCell) // Finding expressed cells for selected genes. Gene->Cell operation.
{
double[] expression = new double[cellNr]; // total expression of each cell.
MT.Loop(0, cellNr, row => {
foreach (int col in selectedGenes)
{
expression[row] += M[row][col];
}
});
double meanExp = expression.Sum() / cellNr;
double maxExp = expression.Max();
double step = (maxExp - meanExp) / maxExprIndex;
this.Genes = selectedGenes.Count;
this.Cells = 0;
for (int i = 0; i < cellNr; i++) // for each cells.
{
double delta = expression[i] - meanExp;
int bIdx = row2bodyIdx[i];
if (delta > 0)
{
short v = Math.Min(maxExprIndex, (short)(delta / step));
BodyList[bIdx].Type = (short)(indexShift + v);
this.Cells++;
expressedId.Add(BodyList[bIdx].Id);
}
else
{
BodyList[bIdx].Type = OrgBodies[bIdx].Type;
}
}
}
else
{
double[] expression = new double[geneNr]; // total expression of each gene.
MT.Loop(0, geneNr, col => {
foreach (int row in selectedCells)
{
expression[col] += M[row][col];
}
});
double meanExp = expression.Sum() / geneNr;
double maxExp = expression.Max();
double step = (maxExp - meanExp) / maxExprIndex;
// Finding expressed genes for selected cells. Cell->Gene operation.
this.Cells = selectedCells.Count;
this.Genes = 0;
for (int i = 0; i < geneNr; i++) // for each gene.
{
double delta = expression[i] - meanExp;
int bIdx = col2bodyIdx[i];
if (delta > 0)
{
int v = Math.Min(maxExprIndex, (int)(delta / step));
BodyList[bIdx].Type = (short)(indexShift + v);
this.Genes++;
expressedId.Add(BodyList[bIdx].Id);
}
else
{
BodyList[bIdx].Type = OrgBodies[bIdx].Type;
}
}
}
}
else // Mark all those cells/genes which have above global average expressions.
{
if (GeneToCell)
{
short[] expressed = new short[cellNr]; // count of expressed genes for each cell.
MT.Loop(0, cellNr, row => {
foreach (int col in selectedGenes)
{
if (M[row][col] > threshold)
{
expressed[row] += 1;
}
}
});
this.Genes = selectedGenes.Count;
this.Cells = expressed.Count(v => v > 0);
for (int i = 0; i < cellNr; i++)
{
short v = Math.Min(maxExprIndex, expressed[i]);
int bIdx = row2bodyIdx[i];
BodyList[bIdx].Type = (short)((v > 0) ? (indexShift + v) : OrgBodies[bIdx].Type);
if (v > 0)
{
expressedId.Add(BodyList[bIdx].Id);
}
}
}
else
{
short[] expressed = new short[geneNr]; // count of expressed cells for each gene.
MT.Loop(0, geneNr, col => {
foreach (int row in selectedCells)
{
if (M[row][col] > threshold)
{
expressed[col] += 1;
}
}
});
this.Cells = selectedCells.Count;
this.Genes = expressed.Count(v => v > 0);
for (int i = 0; i < geneNr; i++)
{
short v = Math.Min(maxExprIndex, expressed[i]);
int bIdx = col2bodyIdx[i];
BodyList[bIdx].Type = (short)((v > 0) ? (indexShift + v) : OrgBodies[bIdx].Type);
if (v > 0)
{
expressedId.Add(BodyList[bIdx].Id);
}
}
}
}
return(expressedId);
}
void PreCalculate()
{
if (dataset != null)
{
var bs = dataset.BodyList;
INumberTable nt = dataset.GetNumberTableView();
int nrows = nt.Rows - nt.Columns;
toIdx = Enumerable.Range(0, bs.Count).Where(i => !bs[i].Disabled).ToArray(); // Indexes of enabled bodies.
int[] selectedRows = toIdx.Where(i => i < nrows).ToArray();
int[] selectedColumns = toIdx.Where(i => i >= nrows).Select(i => i - nrows).ToArray();
if ((selectedColumns.Length == 0) || (selectedRows.Length == 0))
{
throw new TException("No-zero number of genes and cells must be selected!");
}
P = new float[selectedRows.Length][];
MT.Loop(0, P.Length, row => {
float[] R = P[row] = new float[selectedColumns.Length];
double[] dsR = nt.Matrix[selectedRows[row]] as double[];
for (int col = 0; col < selectedColumns.Length; col++)
{
R[col] = (float)dsR[selectedColumns[col]];
}
});
// Reverse toIdx;
int[] rIdx = Enumerable.Repeat(-1, bs.Count).ToArray();
for (int i = 0; i < toIdx.Length; i++)
{
rIdx[toIdx[i]] = i;
}
toIdx = rIdx;
}
float[][] Q = Transpose(P); // Q is the transpose of P.
// Calculate the distance tables for P and Q.
bool isPCor = (metricMode == MetricMode.CorCor) || (metricMode == MetricMode.CorEuc);
bool isQCor = (metricMode == MetricMode.CorCor) || (metricMode == MetricMode.EucCor);
if (isPCor)
{
Normalize(P);
}
if (isQCor)
{
Normalize(Q);
}
using (var gpu = new VisuMap.DxShader.GpuDevice())
using (var cc = gpu.CreateConstantBuffer <ShaderConstant>(0))
using (var sd = gpu.LoadShader("SingleCellAnalysis.DotProduct.cso", Assembly.GetExecutingAssembly())) {
dtP = CallShadere(gpu, cc, sd, P, isPCor);
dtQ = CallShadere(gpu, cc, sd, Q, isQCor);
}
/*
* affinity-propagation enhances structure if colums or rows within clusters are
* light occupied with high randomness. AP however dilutes clusters with few members.
* For instance, singlton gene cluster (with only gene) will suppressed by AP due to
* its aggregation with neighboring genes (which should be consdiered as separate
* clusters.)
*
* ProbagateAffinity(dtQ, P); // Column<->Column affinity => Column->Row affinity
* ProbagateAffinity(dtP, Q); // Row<->Row affinity => Row->Column affinity.
*/
// Calculates the distance between rows and columns into P.
P = AffinityToDistance(P, Q);
Q = null;
var app = SingleCellPlugin.App.ScriptApp;
double linkBias = app.GetPropertyAsDouble("SingleCell.Separation", 1.0);
double pScale = app.GetPropertyAsDouble("SingleCell.CellScale", 1.0);
double qScale = app.GetPropertyAsDouble("SingleCell.GeneScale", 1.0);
//PowerMatrix(P, linkBias);
// Scaling dtP, dtQ to the range of P.
Func <float[][], float> aFct = AverageSqrt;
double av = aFct(P);
pScale *= av / aFct(dtP);
qScale *= av / aFct(dtQ);
ScaleMatrix(dtP, pScale);
ScaleMatrix(dtQ, qScale);
//ScaleMatrix(P, linkBias);
ShiftMatrix(P, (float)(linkBias * av));
}
static float[][] CallShadere(DxShader.GpuDevice gpu, CBuf cc, ComputeShader sd, float[][] M, bool isCorrelation)
{
cc.c.N = M.Length;
int columns = M[0].Length;
const int groupSize = 256;
int distSize = groupSize * cc.c.N;
float[][] dMatrix = new float[M.Length][];
for (int row = 0; row < M.Length; row++)
{
dMatrix[row] = new float[row];
}
int maxColumns = MaxGpuFloats / cc.c.N;
int secSize = Math.Min(columns, (maxColumns > 4096) ? 4096 : (maxColumns - 32));
float[] uploadBuf = new float[cc.c.N * secSize];
using (var dataBuf = gpu.CreateBufferRO(cc.c.N * secSize, 4, 0))
using (var distBuf = gpu.CreateBufferRW(distSize, 4, 0))
using (var distStaging = gpu.CreateStagingBuffer(distBuf)) {
gpu.SetShader(sd);
for (int s0 = 0; s0 < columns; s0 += secSize)
{
int s1 = Math.Min(s0 + secSize, columns);
WriteMarix(gpu, dataBuf, M, s0, s1, uploadBuf);
float[] blockDist = new float[distSize];
for (cc.c.iBlock = 1; cc.c.iBlock < cc.c.N; cc.c.iBlock += groupSize)
{
cc.c.columns = s1 - s0;
cc.Upload();
gpu.Run(groupSize);
int iBlock2 = Math.Min(cc.c.iBlock + groupSize, cc.c.N);
int bSize = (iBlock2 - cc.c.iBlock) * (iBlock2 + cc.c.iBlock - 1) / 2;
gpu.ReadRange <float>(blockDist, 0, distStaging, distBuf, bSize);
int offset = 0;
for (int row = cc.c.iBlock; row < iBlock2; row++)
{
float[] R = dMatrix[row];
for (int k = 0; k < row; k++)
{
R[k] += blockDist[offset + k];
}
offset += row;
}
Application.DoEvents();
}
}
}
if (isCorrelation)
{
MT.ForEach(dMatrix, R => {
for (int col = 0; col < R.Length; col++)
{
R[col] = 1.0f - R[col];
}
});
}
else // Euclidean distance is wanted.
{
float[] norm2 = new float[M.Length];
Array.Clear(norm2, 0, M.Length);
int L = M[0].Length;
MT.ForEach(M, (R, row) => {
float sumSquared = 0.0f;
for (int col = 0; col < L; col++)
{
sumSquared += R[col] * R[col];
}
norm2[row] = sumSquared;
});
MT.Loop(1, M.Length, row => {
float[] R = dMatrix[row];
for (int col = 0; col < row; col++)
{
R[col] = (float)Math.Sqrt(Math.Abs(norm2[row] + norm2[col] - 2 * R[col]));
}
});
}
return(dMatrix);
}