public WaveletD4Transform(int dataDim)
{
dim = HaarTransform.RoundDimension(dataDim);
InitializeConstants();
D4 = WaveTransforms.App.ScriptApp.New.NumberTable(dim, dim);
// Algorithm adapated from: http://www.bearcave.com/misl/misl_tech/wavelets/daubechies/daub.java
for (int row = 0; row < dim; row++)
{
double[] v = (D4.Matrix as double[][])[row];
Array.Clear(v, 0, v.Length);
v[row] = 1.0;
for (int n = v.Length; n >= 4; n >>= 1)
{
TransformD4(v, n);
}
}
int interval = dim / 16;
for (int k = 0; k < dim; k++)
{
D4.RowSpecList[k].Type = D4.ColumnSpecList[k].Group = (short)(k / interval);
}
List <int> rows = new List <int>();
for (int row = 0; row < dataDim; row++)
{
rows.Add(row);
}
D4 = D4.SelectRows(rows);
}
/// <summary>
/// Perform CCA analysis on columns of two number tables.
/// </summary>
/// <param name="tableX">A number table.</param>
/// <param name="tableY">A number table.</param>
/// <returns>A PlsResult structure.</returns>
/// <remarks>This method finds directions for tableX and tableY; so that these two tables will
/// have maximal correlation in those directions.</remarks>
public PlsResult DoCCA(INumberTable tableX, INumberTable tableY)
{
double[][] X = (double[][])tableX.Matrix;
double[][] Y = (double[][])tableY.Matrix;
double[][] Cxy = Covariance(X, Y);
double[][] Cyx = Covariance(Y, X);
IMathAdaptor math = MultivariateAnalysisPlugin.App.GetMathAdaptor();
double[][] rCxx = math.InvertMatrix(Covariance(X, X));
double[][] rCyy = math.InvertMatrix(Covariance(Y, Y));
MakeSymmetric(rCxx);
MakeSymmetric(rCyy);
double[][] A = MatrixProduct(rCxx, Cxy);
double[][] B = MatrixProduct(rCyy, Cyx);
double[][] Cx = MatrixProduct(A, B);
double[][] Cy = MatrixProduct(B, A);
double[][] Wx, Wy;
double[] Rx, Ry;
math.EigenDecomposition(Cx, out Wx, out Rx);
math.EigenDecomposition(Cy, out Wy, out Ry);
//
// Rx and Ry should be the equal upto permutation and dimension.
// Wy can be calculated from Wx by Ry = sqrt(1/Rx) * B * Rx
//
/*
* double x0 = Math.Sqrt(Rx[0]);
* double x1 = Math.Sqrt(Rx[1]);
* double x2 = Math.Sqrt(Rx[2]);
*
* double y0 = Math.Sqrt(Ry[0]);
* double y1 = Math.Sqrt(Ry[1]);
* double y2 = Math.Sqrt(Ry[2]);
*
* 0.7165 0.4906 0.2668
*/
SortEigenVectors(Wx, Rx);
SortEigenVectors(Wy, Ry);
//ValidateMatrix(Wx);
//ValidateMatrix(Wy);
PlsResult ret = new PlsResult();
ret.ProjectionX = EigenProjection(tableX, Wx);
ret.ProjectionY = EigenProjection(tableY, Wy);
ret.EigenVectorsX = Wx;
ret.EigenVectorsY = Wy;
ret.EigenValuesX = Rx;
ret.EigenValuesY = Ry;
return(ret);
}
/// <summary>
/// Projects the rows of a data table to a set of eigenvectors.
/// </summary>
/// <param name="dataTable">The input data table.</param>
/// <param name="eigenVectors">The eigenvectors as row vectors.</param>
/// <returns>An INumberTable with row vectors as the projection.</returns>
INumberTable EigenProjection(INumberTable dataTable, double[][] eigenVectors)
{
int rows = dataTable.Rows;
int columns = dataTable.Columns;
int outDim = eigenVectors.Length;
INumberTable outTable = MultivariateAnalysisPlugin.App.ScriptApp.New.NumberTable(rows, outDim);
double[][] outMatrix = (double[][])outTable.Matrix;
double[][] m = (double[][])dataTable.Matrix;
for (int row = 0; row < rows; row++)
{
for (int j = 0; j < outDim; j++)
{
double v = 0.0;
for (int col = 0; col < columns; col++)
{
v += m[row][col] * eigenVectors[j][col];
}
outMatrix[row][j] = v;
}
}
for (int row = 0; row < rows; row++)
{
outTable.RowSpecList[row].CopyFrom(dataTable.RowSpecList[row]);
}
return(outTable);
}
/// <summary>
/// Perform PLS analysis on columns of two number tables.
/// </summary>
/// <param name="tableX">A number table.</param>
/// <param name="tableY">A number table.</param>
/// <returns>A PlsResult structure.</returns>
/// <remarks>This method finds directions for tableX and tableY; so that these two tables will
/// have maximal covariance in those directions.</remarks>
public PlsResult DoPLS(INumberTable tableX, INumberTable tableY)
{
double[][] Cxy = Covariance((double[][])tableX.Matrix, (double[][])tableY.Matrix);
double[][] Cyx = Covariance((double[][])tableY.Matrix, (double[][])tableX.Matrix);
double[][] CxyCyx = MatrixProduct(Cxy, Cyx);
double[][] CyxCxy = MatrixProduct(Cyx, Cxy);
IMathAdaptor math = MultivariateAnalysisPlugin.App.GetMathAdaptor();
double[][] Wx, Wy;
double[] Rx, Ry;
// These two matrix might be slightly asymmetric due to calculation errors.
MakeSymmetric(CxyCyx);
MakeSymmetric(CyxCxy);
math.EigenDecomposition(CxyCyx, out Wx, out Rx);
math.EigenDecomposition(CyxCxy, out Wy, out Ry);
SortEigenVectors(Wx, Rx);
SortEigenVectors(Wy, Ry);
PlsResult ret = new PlsResult();
ret.ProjectionX = EigenProjection(tableX, Wx);
ret.ProjectionY = EigenProjection(tableY, Wy);
ret.EigenVectorsX = Wx;
ret.EigenVectorsY = Wy;
ret.EigenValuesX = Rx;
ret.EigenValuesY = Ry;
return(ret);
}
void AddCluster(INumberTable nt, double[] expLevel, int count, short type)
{
int offset = (nt.Tag == null) ? 0 : (int)(nt.Tag);
nt.Tag = offset + count;
double[][] m = nt.Matrix as double[][];
int columns = expLevel.Length;
for (int col = 0; col < columns; col++)
{
double mean = expLevel[col];
if (mean > 0)
{
double[] v = NewGaussianSampling(count, expLevel[col]);
for (int row = 0; row < count; row++)
{
m[offset + row][col] = v[row];
}
}
}
for (int row = offset; row < (offset + count); row++)
{
nt.RowSpecList[row].Type = type;
}
}
public INumberTable Filter(INumberTable inTable, double lowFreq, double highFreq)
{
// Centralize the table.
inTable = inTable.Clone();
IList <IList <double> > m = inTable.Matrix;
for (int row = 0; row < inTable.Rows; row++)
{
for (int col = 0; col < inTable.Columns; col++)
{
m[row][col] -= columnMean[col];
}
}
INumberTable outTable = WalshTransform.Filter(inTable, lowFreq, highFreq, baseTable.Columns, baseTable);
m = outTable.Matrix;
for (int row = 0; row < outTable.Rows; row++)
{
for (int col = 0; col < outTable.Columns; col++)
{
m[row][col] += columnMean[col];
}
}
return(outTable);
}
void ValidateMatrix(double[][] m)
{
INumberTable nt = MultivariateAnalysisPlugin.App.ScriptApp.New.NumberTable(m);
INumberTable ntT = nt.Transpose2();
INumberTable I = nt.Multiply(ntT);
I.ShowAsTable();
}
public FourierTransform(int dimension)
{
tDim = dimension;
matrixReal = WaveTransforms.App.ScriptApp.New.NumberTable(tDim, tDim);
matrixImg = WaveTransforms.App.ScriptApp.New.NumberTable(tDim, tDim);
double[][] mr = matrixReal.Matrix as double[][];
double[][] mi = matrixImg.Matrix as double[][];
double w = -2 * Math.PI / dimension;
double f = 1 / Math.Sqrt(tDim);
IList <IRowSpec> rowListReal = matrixReal.RowSpecList;
IList <IRowSpec> rowListImg = matrixImg.RowSpecList;
for (int i = 0; i < tDim; i++)
{
for (int j = 0; j <= i; j++)
{
double wij = i * j * w;
mr[i][j] = mr[j][i] = f * Math.Cos(wij);
mi[i][j] = mi[j][i] = f * Math.Sin(wij);
}
short freq = (short)((i < tDim / 2) ? i : (tDim - i));
if (tDim % 2 == 0)
{
rowListReal[i].Type = freq;
rowListImg[i].Type = freq;
if (i == tDim / 2)
{
rowListImg[i].Type = 0;
}
}
else
{
rowListReal[i].Type = freq;
rowListImg[i].Type = freq;
if (i == tDim / 2)
{
rowListImg[i].Type = rowListReal[i].Type = (short)(tDim / 2);
}
}
matrixReal.ColumnSpecList[i].Group = rowListReal[i].Type;
matrixImg.ColumnSpecList[i].Group = rowListImg[i].Type;
}
// indexes to select non-duplicate part of the fourier matrix.
selectReal = new int[tDim / 2 + 1];
for (int i = 0; i < selectReal.Length; i++)
{
selectReal[i] = i;
}
selectImg = new int[(tDim % 2 == 0) ? (tDim / 2 - 1) : (tDim / 2)];
for (int i = 0; i < selectImg.Length; i++)
{
selectImg[i] = i + 1;
}
}
/// <summary>
/// Multiply two matrix with padding and repeating if dimensions don't match.
/// </summary>
/// <param name="left">The left matrix.</param>
/// <param name="right">The right matrix.</param>
/// <returns>The product of two matrix.</returns>
public static INumberTable MatrixProduct(INumberTable left, INumberTable right)
{
int dim = right.Rows; // The dimension of the transformation.
int repeats = left.Columns / dim;
if (left.Columns % dim > 0)
{
repeats++;
}
int outColumns = repeats * right.Columns;
INumberTable prod = WaveTransforms.App.ScriptApp.New.NumberTable(left.Rows, outColumns);
double[][] P = (double[][])prod.Matrix;
double[][] L = (double[][])left.Matrix;
double[][] R = (double[][])right.Matrix;
for (int row = 0; row < prod.Rows; row++)
{
double[] Lrow = L[row];
for (int col = 0; col < prod.Columns; col++)
{
int col2 = col % right.Columns;
int k0 = col / right.Columns * dim;
double v = 0;
for (int k = 0; k < dim; k++)
{
if ((k0 + k) < Lrow.Length)
{
v += Lrow[k0 + k] * R[k][col2];
}
}
P[row][col] = v;
}
}
for (int row = 0; row < left.Rows; row++)
{
prod.RowSpecList[row].CopyFrom(left.RowSpecList[row]);
}
IList <IColumnSpec> pSpecList = prod.ColumnSpecList;
IList <IColumnSpec> rSpecList = right.ColumnSpecList;
for (int col = 0; col < prod.Columns; col++)
{
IColumnSpec cs = pSpecList[col];
cs.CopyFrom(rSpecList[col % right.Columns]);
int n = col / right.Columns;
if (n > 0)
{
cs.Id += "_" + n;
}
}
return(prod);
}
public void SetOperationMode()
{
operationMode = true;
var app = DataModeling.App.ScriptApp;
numTable = (app.SelectedItems.Count == 0) ? app.GetNumberTable() : app.GetSelectedNumberTable();
this.Text = "Operation Node List";
this.listView1.Columns[0].Text = "Name";
this.listView1.Columns[1].Text = "Type";
}
IList <int> ToIndexList(INumberTable nTable, bool attributeMode, IList <string> itemList)
{
if (attributeMode)
{
return(nTable.IndexOfColumns(itemList));
}
else
{
return(nTable.IndexOfRows(itemList));
}
}
public void SetExpression(INumberTable numTable, IList <IBody> bodyList, IList <IBody> orgBodies, int N)
{
NumTable = numTable;
OrgBodies = orgBodies;
BodyList = bodyList;
this.N = N;
row2bodyIdx = Enumerable.Range(0, N).Where(i => !OrgBodies[i].Disabled).ToArray();
col2bodyIdx = Enumerable.Range(N, OrgBodies.Count - N).Where(i => !OrgBodies[i].Disabled).ToArray();
MeanExpression = ExpThreshold();
}
public void Duplicate(INumberTable nTable, bool attributeMode, IList <string> itemList)
{
if (attributeMode)
{
nTable.AppendColumns(nTable.SelectColumnsById(itemList));
}
else
{
nTable.Append(nTable.SelectRowsById(itemList));
}
}
public INumberTable Transform(INumberTable inTable)
{
INumberTable rMatrix = matrixReal.SelectColumns(selectReal);
INumberTable iMatrix = matrixImg.SelectColumns(selectImg);
IList <IColumnSpec> cs = iMatrix.ColumnSpecList;
for (int i = 0; i < cs.Count; i++)
{
cs[i].Id += "_i";
}
return(MatrixProduct(inTable, rMatrix.AppendColumns(iMatrix)));
}
public void Delete(INumberTable nTable, bool attributeMode, IList <string> itemList)
{
IList <int> idxList = ToIndexList(nTable, attributeMode, itemList);
if (attributeMode)
{
nTable.RemoveColumns(idxList);
}
else
{
nTable.RemoveRows(idxList);
}
}
public IPcaView Show()
{
INumberTable nt = DataGenerator.App.ScriptApp.New.NumberTable(bodies.Count, 3);
for (int row = 0; row < nt.Rows; row++)
{
nt.Matrix[row][0] = bodies[row].X;
nt.Matrix[row][1] = bodies[row].Y;
nt.Matrix[row][2] = bodies[row].Z;
nt.RowSpecList[row].Type = (short)bodies[row].Type;
}
return(nt.ShowPcaView());
}
public void Scale(INumberTable nTable, bool attributeMode, IList <string> itemList, double factor)
{
double[][] m = (double[][])nTable.Matrix;
IList <int> idxList = ToIndexList(nTable, attributeMode, itemList);
if (factor == 0)
{
double maxValue = double.MinValue;
foreach (int i in idxList)
{
if (attributeMode)
{
for (int row = 0; row < nTable.Rows; row++)
{
maxValue = Math.Max(maxValue, m[row][i]);
}
}
else
{
for (int col = 0; col < nTable.Columns; col++)
{
maxValue = Math.Max(maxValue, m[i][col]);
}
}
}
if (maxValue != 0)
{
factor = Math.Abs(nTable.MaximumValue()) / Math.Abs(maxValue);
}
}
foreach (int i in idxList)
{
if (attributeMode)
{
for (int row = 0; row < nTable.Rows; row++)
{
m[row][i] *= factor;
}
}
else
{
for (int col = 0; col < nTable.Columns; col++)
{
m[i][col] *= factor;
}
}
}
}
public PcaTransform(INumberTable numberTable)
{
baseTable = numberTable.GetPcaEigenvectors(null, 0).Transpose();
columnMean = new double[numberTable.Columns];
IList <IList <double> > m = numberTable.Matrix;
for (int col = 0; col < numberTable.Columns; col++)
{
columnMean[col] = 0;
for (int row = 0; row < numberTable.Rows; row++)
{
columnMean[col] += m[row][col];
}
columnMean[col] /= numberTable.Rows;
}
}
public static IPcaView Show(IList <Vector4D> points, int dimension)
{
INumberTable nt = DataGenerator.App.ScriptApp.New.NumberTable(points.Count, dimension);
for (int row = 0; row < nt.Rows; row++)
{
nt.Matrix[row][0] = points[row].x;
nt.Matrix[row][1] = points[row].y;
if (dimension == 4)
{
nt.Matrix[row][2] = points[row].z;
nt.Matrix[row][3] = points[row].w;
}
nt.RowSpecList[row].Type = (short)points[row].type;
}
return(nt.ShowPcaView());
}
public void Initialize(IDataset dataset, XmlElement filterNode)
{
INumberTable numTable = dataset.GetNumberTable();
int N = numTable.Rows;
double[][] L = Matrix(N, N);
int offset = numTable.Columns - N;
for (int i = 0; i < N; i++)
{
for (int j = 0; j < N; j++)
{
if (i != j)
{
L[i][j] = -numTable.Matrix[i][offset + j];
}
else
{
L[i][j] = 0;
}
}
}
for (int i = 0; i < N; i++)
{
double rowSum = 0;
for (int j = 0; j < N; j++)
{
rowSum += L[i][j];
}
L[i][i] = -rowSum;
}
IMathAdaptor math = GraphMetrics.App.GetMathAdaptor();
double[][] H = math.InvertMatrix(L);
d = Matrix(N, N);
for (int i = 0; i < N; i++)
{
for (int j = 0; j < N; j++)
{
d[i][j] = H[i][i] + H[j][j] - H[i][j] - H[j][i];
}
}
}
public HaarTransform(int dataDim)
{
dim = RoundDimension(dataDim);
haar = WaveTransforms.App.ScriptApp.New.NumberTable(dim, dim);
// Algorithm adapated from: http://www.cs.ucf.edu/~mali/haar/.
double sqrt2 = 1 / Math.Sqrt(2.0);
double[] vp = new double[dim];
for (int row = 0; row < dim; row++)
{
double[] v = (haar.Matrix as double[][])[row];
Array.Clear(v, 0, v.Length);
v[row] = 1.0;
Array.Clear(vp, 0, vp.Length);
int w = dim;
while (w > 1)
{
w /= 2;
for (int i = 0; i < w; i++)
{
vp[i] = (v[2 * i] + v[2 * i + 1]) * sqrt2;
vp[i + w] = (v[2 * i] - v[2 * i + 1]) * sqrt2;
}
Array.Copy(vp, v, 2 * w);
}
}
int interval = dim / 16;
for (int k = 0; k < dim; k++)
{
haar.RowSpecList[k].Type = haar.ColumnSpecList[k].Group = (short)(k / interval);
}
List <int> rows = new List <int>();
for (int row = 0; row < dataDim; row++)
{
rows.Add(row);
}
haar = haar.SelectRows(rows);
}
public static INumberTable Filter(INumberTable inTable, double lowFreq, double highFreq, int dimension, INumberTable baseTable)
{
int lowIdx = (int)(lowFreq * dimension);
int hiIdx = (int)(highFreq * dimension);
List <int> columns = new List <int>();
for (int i = 0; i < dimension; i++)
{
if ((i >= lowIdx) && (i <= hiIdx))
{
columns.Add(i);
}
}
INumberTable B = baseTable.SelectColumns(columns);
INumberTable Bt = B.Transpose2();
int b = B.Columns;
int r = inTable.Rows;
int m = inTable.Columns;
INumberTable outTable = null;
//Depending on the size of m relative to r and b, we can
//optimize the calculation by arrange the matrix multiplication.
if (2 * r * b < m * (r + b))
{
// The complexity in this arrangement is 2*m*r*b.
outTable = FourierTransform.MatrixProduct(FourierTransform.MatrixProduct(inTable, B), Bt);
}
else
{
// The complexity in this arrangement is m*m*(r+b).
outTable = FourierTransform.MatrixProduct(inTable, FourierTransform.MatrixProduct(B, Bt));
}
IList <IColumnSpec> oSpecList = outTable.ColumnSpecList;
IList <IColumnSpec> iSpecList = inTable.ColumnSpecList;
for (int col = 0; col < outTable.Columns; col++)
{
oSpecList[col].CopyFrom(iSpecList[col]);
}
return(outTable);
}
void PreCalculate() {
if (dataset == null)
return;
// Extract the relevant data table.
var bs = dataset.BodyList;
INumberTable nt = dataset.GetNumberTableView();
toIdx = Enumerable.Range(0, bs.Count).Where(i => !bs[i].Disabled).ToArray(); // Indexes of enabled bodies.
int N = nt.Rows - pcaSamples;
int[] enabledRows = toIdx.Where(i => i < N).ToArray();
if (enabledRows.Length == 0)
throw new TException("No data available!");
P = new float[enabledRows.Length][];
MT.Loop(0, P.Length, row => {
float[] R = P[row] = new float[nt.Columns];
double[] dsR = nt.Matrix[enabledRows[row]] as double[];
for (int col = 0; col < nt.Columns; col++)
R[col] = (float)dsR[col];
});
// Reverse toIdx;
int[] rIdx = Enumerable.Repeat(-1, bs.Count).ToArray();
for (int i = 0; i < toIdx.Length; i++) rIdx[toIdx[i]] = i;
toIdx = rIdx;
using (var gpu = new VisuMap.DxShader.GpuDevice())
dtP = DualAffinity.DotProduct(gpu, P, false);
float[] singValues = new float[pcaMax];
float[][] PC = FastPca.DoFastReduction(P, pcaMax, singValues, true);
P = VisuMap.MathUtil.NewMatrix<float>(PC.Length, pcaSamples); // P now links data points with the injected points on the main PCA axis.
float span = 4.0f * singValues[0];
stepSize = span / (pcaSamples - 1);
float x0 = - 0.5f * span;
MT.ForEach(PC, (R, row) => {
double yy = R.Skip(1).Sum(v => v * v);
for (int col = 0; col < pcaSamples; col++) {
double x = R[0] - (x0 + col * stepSize);
P[row][col] = (float)Math.Sqrt(x * x + yy);
}
});
}
public void Normalize(INumberTable nTable, bool attributeMode, IList <string> itemList)
{
double[][] m = (double[][])nTable.Matrix;
IList <int> idxList = ToIndexList(nTable, attributeMode, itemList);
if (attributeMode)
{
foreach (int col in idxList)
{
double vMax = double.MinValue;
double vMin = double.MaxValue;
for (int row = 0; row < nTable.Rows; row++)
{
vMax = Math.Max(vMax, m[row][col]);
vMin = Math.Min(vMin, m[row][col]);
}
double vRange = vMax - vMin;
for (int row = 0; row < nTable.Rows; row++)
{
m[row][col] = (vRange == 0) ? 0 : (m[row][col] - vMin) / vRange;
}
}
}
else
{
foreach (int row in idxList)
{
double vMax = double.MinValue;
double vMin = double.MaxValue;
for (int col = 0; col < nTable.Columns; col++)
{
vMax = Math.Max(vMax, m[row][col]);
vMin = Math.Min(vMin, m[row][col]);
}
double vRange = vMax - vMin;
for (int col = 0; col < nTable.Columns; col++)
{
m[row][col] = (vRange == 0) ? 0 : (m[row][col] - vMin) / vRange;
}
}
}
}
public void Logarithmic(INumberTable nTable, bool attributeMode, IList <string> itemList)
{
double[][] m = (double[][])nTable.Matrix;
foreach (int i in ToIndexList(nTable, attributeMode, itemList))
{
if (attributeMode)
{
for (int row = 0; row < nTable.Rows; row++)
{
m[row][i] = Math.Log(1 + Math.Abs(m[row][i]));
}
}
else
{
for (int col = 0; col < nTable.Columns; col++)
{
m[i][col] = Math.Log(1 + Math.Max(0, m[i][col]));
}
}
}
}
public void SetExpressionTable(INumberTable numTable, IForm featureView)
{
if ((featureView is IMapSnapshot) || (featureView is IMdsCluster))
{
featureMap = featureView;
}
else
{
MsgBox.Alert("Only parnet window is not a map-snapshot or mds-cluster view.");
return;
}
NumTable = numTable;
OrgBodies = featureMap.BodyList;
featureMap.GlyphSet = "Ordered Glyphs";
featureMap.Redraw();
minExpression = double.MaxValue;
double maxExpression = double.MinValue;
MT.ForEach(NumTable.Matrix, R => {
double minV = double.MaxValue;
double maxV = double.MinValue;
foreach (var v in R)
{
minV = Math.Min(minV, v);
maxV = Math.Max(maxV, v);
}
lock (this) {
minExpression = Math.Min(minExpression, minV);
maxExpression = Math.Max(maxExpression, maxV);
}
});
stepExpression = (maxExpression - minExpression) / 16;
SingleCellPlugin.App.ItemsSelected += App_ItemsSelected;
featureView.TheForm.FormClosing += (s, e) => {
SingleCellPlugin.App.ItemsSelected -= App_ItemsSelected;
};
}
public void Logicle(INumberTable nTable, bool attributeMode, IList <string> itemList)
{
double[][] m = (double[][])nTable.Matrix;
double T = 262144;
double W = 1.0;
double M = 4.5;
string[] settings = DataCleansing.App.ScriptApp.GetProperty(
"DataCleansing.Logicle.Settings", "262144; 1.0; 4.5").Split(';');
if (settings.Length == 3)
{
double.TryParse(settings[0], out T);
double.TryParse(settings[1], out W);
double.TryParse(settings[2], out M);
}
FastLogicle fastLogicle = new FastLogicle(T, W, M);
double maxVal = fastLogicle.MaxValue;
double minVal = fastLogicle.MinValue;
foreach (int i in ToIndexList(nTable, attributeMode, itemList))
{
if (attributeMode)
{
for (int row = 0; row < nTable.Rows; row++)
{
m[row][i] = M * fastLogicle.scale(Math.Min(maxVal, Math.Max(minVal, m[row][i])));
}
}
else
{
for (int col = 0; col < nTable.Columns; col++)
{
m[i][col] = M * fastLogicle.scale(Math.Min(maxVal, Math.Max(minVal, m[i][col])));
}
}
}
}
public PcaTransform NewPca(INumberTable numberTable)
{
return(new PcaTransform(numberTable));
}
public bool ImportFile(string fileName)
{
using (FileStream f = new FileStream(fileName, FileMode.Open, FileAccess.Read))
using (BinaryReader br = new BinaryReader(f)) {
byte[] magic = br.ReadBytes(6);
if (magic[0] != 0x93)
{
return(false);
}
byte[] version = br.ReadBytes(2);
int headLen = 0;
if (version[0] == 1)
{
headLen = br.ReadInt16();
}
else
{
headLen = br.ReadInt32();
}
byte[] bHead = br.ReadBytes(headLen);
string sHead = Encoding.UTF8.GetString(bHead);
sHead = sHead.Trim(new char[] { '{', ',', '}' });
string[] fs = sHead.Split(new char[] { ' ', ',', ':', '\'', ')', '(' }, StringSplitOptions.RemoveEmptyEntries);
NumpyDtype dtype = NumpyDtype.DT_Unknown;
bool isFortranOrder = false;
List <int> dims = new List <int>();
for (int i = 0; i < fs.Length; i += 2)
{
switch (fs[i])
{
case "descr":
switch (fs[i + 1])
{
case "<f4":
dtype = NumpyDtype.DT_Float32;
break;
case "<f8":
dtype = NumpyDtype.DT_Float64;
break;
case "<i4":
dtype = NumpyDtype.DT_Int32;
break;
default:
dtype = NumpyDtype.DT_Unknown;
break;
}
break;
case "fortran_order":
isFortranOrder = (fs[i + 1] == "True");
break;
case "shape":
dims.Add(int.Parse(fs[i + 1]));
for (int j = i + 2; j < fs.Length; j++)
{
if (char.IsDigit(fs[j][0]))
{
dims.Add(int.Parse(fs[j]));
}
}
break;
}
}
if (dtype == NumpyDtype.DT_Unknown)
{
MessageBox.Show("Only float32, float64 and int32 data types are supported!");
return(false);
}
if (dims.Count == 0)
{
MessageBox.Show("No shape defined!");
return(false);
}
var appNew = DataModeling.App.ScriptApp.New;
if (dims.Count == 2)
{
INumberTable nt = null;
if (isFortranOrder)
{
nt = appNew.NumberTable(dims[1], dims[0]);
for (int row = 0; row < dims[1]; row++)
{
ReadRow(nt.Matrix[row] as double[], br, dtype);
}
nt.Transpose();
}
else
{
nt = appNew.NumberTable(dims[0], dims[1]);
for (int row = 0; row < dims[0]; row++)
{
ReadRow(nt.Matrix[row] as double[], br, dtype);
}
}
if (DataModeling.App.ScriptApp.Dataset != null)
{
var bs = DataModeling.App.ScriptApp.Dataset.BodyListEnabled();
if (nt.Rows != bs.Count)
{
bs = DataModeling.App.ScriptApp.Map.SelectedBodies;
if (nt.Rows != bs.Count)
{
bs = null;
}
}
if (bs != null)
{
for (int i = 0; i < nt.Rows; i++)
{
nt.RowSpecList[i].CopyFromBody(bs[i]);
}
}
}
var hm = nt.ShowHeatMap();
hm.Title = "HeatMap: " + fileName;
}
else
{
int len = 1;
foreach (int d in dims)
{
len *= d;
}
float[] values = new float[len];
for (int i = 0; i < len; i++)
{
values[i] = (float)NextValue(br, dtype);
}
var bb = appNew.BigBarView(values).Show();
bb.Title = "BigBar View: " + fileName;
}
}
return(true);
}
public bool ImportFile0(string fileName)
{
IVisuMap app = CustomImporter.App.ScriptApp;
if (!fileName.ToLower().EndsWith(".fcs"))
{
return(false); // Let other import handle it.
}
INumberTable dataTable = null;
IFreeTable textTable = null;
bool compensated = false;
bool logScaled = false;
using (StreamReader sr = new StreamReader(fileName)) {
//
// Read the text segment.
//
char[] header = new char[42];
sr.ReadBlock(header, 0, header.Length);
int textBegin = int.Parse(new string(header, 10, 8));
int textEnd = int.Parse(new string(header, 18, 8));
int beginData = int.Parse(new string(header, 26, 8));
int endData = int.Parse(new string(header, 34, 8));
char[] line = new char[textEnd + 4];
sr.ReadBlock(line, header.Length, line.Length - header.Length);
string textSeg = new string(line, textBegin + 1, textEnd - textBegin);
char delimiter = line[textBegin];
textTable = app.New.FreeTable();
textTable.AddColumn("Value", false);
string[] textFields = textSeg.Split(delimiter);
textTable.AddRows("P", textFields.Length / 2);
IList <IRowSpec> rowSpecList = textTable.RowSpecList;
for (int row = 0; row < textTable.Rows; row++)
{
rowSpecList[row].Id = textFields[2 * row];
textTable.Matrix[row][0] = textFields[2 * row + 1];
}
//
// Read in the data segement
//
fcsInfo = new FcsInfo(textTable, header);
if ((beginData == 0) && (textTable.IndexOfRow("$BEGINDATA") > 0))
{
beginData = int.Parse(textTable.Matrix[textTable.IndexOfRow("$BEGINDATA")][0]);
}
if ((endData == 0) && (textTable.IndexOfRow("$ENDDATA") > 0))
{
endData = int.Parse(textTable.Matrix[textTable.IndexOfRow("$ENDDATA")][0]);
}
dataTable = app.New.NumberTable(fcsInfo.Rows, fcsInfo.Columns);
using (BinaryReader br = new BinaryReader(sr.BaseStream)) {
br.BaseStream.Seek(beginData, SeekOrigin.Begin);
Byte[] buf4 = new byte[4];
Byte[] buf8 = new byte[8];
int bitOffset = 0;
for (int row = 0; row < fcsInfo.Rows; row++)
{
for (int col = 0; col < fcsInfo.Columns; col++)
{
ColumnInfo ci = fcsInfo.ColumnInfo[col];
Byte[] data = ReadBits(br, ci.Bits, ref bitOffset);
if (data.Length < ci.Bytes)
{
row = fcsInfo.Rows; // enforce premature loop-end.
break;
}
Byte[] buf = (fcsInfo.DataType == FcsInfo.DataTypes.Double) ? buf8 : buf4;
Array.Clear(buf, 0, buf.Length);
Array.Copy(data, 0, buf, 0, ci.Bytes);
if (!fcsInfo.BigEndian)
{
// Intel CPU expects BigEndian format.
Array.Reverse(buf, 0, ci.Bytes);
}
switch (fcsInfo.DataType)
{
case FcsInfo.DataTypes.Integer:
dataTable.Matrix[row][col] = (BitConverter.ToUInt32(buf, 0) & ci.RangeMask);
break;
case FcsInfo.DataTypes.Float:
dataTable.Matrix[row][col] = BitConverter.ToSingle(buf, 0);
break;
case FcsInfo.DataTypes.Double:
dataTable.Matrix[row][col] = BitConverter.ToDouble(buf, 0);
break;
}
}
}
}
}
// Post processing
for (int col = 0; col < fcsInfo.Columns; col++)
{
IColumnSpec cs = dataTable.ColumnSpecList[col];
ColumnInfo cInfo = fcsInfo.ColumnInfo[col];
cs.Id = cInfo.ShortName;
//cs.Name = cInfo.Name + ( cInfo.IsLinear ? ".Lin" : ".Log");
cs.Name = cInfo.Name;
if ((cs.Id == "TIME") || (cs.Id == "TIME1"))
{
int timeStepIdx = textTable.IndexOfRow("$TIMESTEP");
if (timeStepIdx >= 0)
{
double timeStep = double.Parse(textTable.Matrix[timeStepIdx][0]);
for (int row = 0; row < fcsInfo.Rows; row++)
{
dataTable.Matrix[row][col] *= timeStep;
}
}
}
}
IList <IRowSpec> rSpecList = dataTable.RowSpecList;
for (int row = 0; row < fcsInfo.Rows; row++)
{
rSpecList[row].Id = row.ToString();
}
FileInfo fInfo = new FileInfo(fileName);
string shortName = fInfo.Name.Substring(0, fInfo.Name.LastIndexOf('.'));
if (saveMetaInfo >= 1)
{
SaveParameterTable(textTable, shortName);
if (saveMetaInfo >= 2)
{
textTable.SaveAsDataset(shortName + " (Text Seg)", "Text segement of data table " + shortName);
}
}
if (autoCompensation)
{
try {
string sMatrix = null;
for (int i = 0; i < textTable.Rows; i++)
{
string id = textTable.RowSpecList[i].Id.ToLower();
if (id.StartsWith("$"))
{
id = id.Substring(1);
}
if ((id == "spill") || (id == "spillover"))
{
sMatrix = textTable.Matrix[i][0];
break;
}
}
if (sMatrix == null)
{
throw new Exception(""); // silently ignore compensation as no compensation matrix available.
}
string[] fs = sMatrix.Split(',');
int dimension = int.Parse(fs[0]);
if (fs.Length != (dimension * dimension + dimension + 1))
{
throw new Exception("Invalid spill over matrix.");
}
INumberTable cMatrix = app.New.NumberTable(dimension, dimension);
List <string> parList = fs.Skip(1).Take(dimension).ToList();
int idx;
if (parList.Count(id => int.TryParse(id, out idx)) == parList.Count)
{
// The columns are specified by a list of indexes. We convert them here to ids
for (int i = 0; i < parList.Count; i++)
{
parList[i] = dataTable.ColumnSpecList[int.Parse(parList[i]) - 1].Id; // index starts with 1 !
}
}
for (int col = 0; col < cMatrix.Columns; col++)
{
cMatrix.ColumnSpecList[col].Id = parList[col];
}
int fsIdx = dimension + 1;
for (int row = 0; row < cMatrix.Rows; row++)
{
for (int col = 0; col < cMatrix.Columns; col++)
{
cMatrix.Matrix[row][col] = double.Parse(fs[fsIdx++]);
}
}
var cData = dataTable.SelectColumnsById(parList);
if (cData.Columns != parList.Count)
{
if (dataTable.ColumnSpecList.Count(cl => cl.Id.Equals("<" + parList[0] + ">")) == 1)
{
// siliently ignore aready compensated data.
throw new Exception("");
}
else
{
throw new Exception("Invalid spill over matrix: unknown names.");
}
}
var math = CustomImporter.App.GetMathAdaptor();
var m = math.InvertMatrix((double[][])cMatrix.Matrix);
for (int row = 0; row < cMatrix.Rows; row++)
{
cMatrix.Matrix[row] = m[row];
}
cData = cData.Multiply(cMatrix); // perform the comensation with the inverse matrix of the spill over matrix.
cData.CopyValuesTo(dataTable);
compensated = true;
} catch (Exception ex) {
if (ex.Message != "")
{
MessageBox.Show("Cannot perform compensation: " + ex.Message);
}
}
}
if (logTransform)
{
double[][] m = (double[][])dataTable.Matrix;
double T = 262144;
double W = 1.0;
double M = 4.5;
string[] settings = app.GetProperty(
"CustomImporter.Logicle.Settings", "262144; 1.0; 4.5").Split(';');
if (settings.Length == 3)
{
double.TryParse(settings[0], out T);
double.TryParse(settings[1], out W);
double.TryParse(settings[2], out M);
}
var fastLogicle = new VisuMap.DataCleansing.FastLogicle(T, W, M);
double maxVal = fastLogicle.MaxValue;
double minVal = fastLogicle.MinValue;
for (int col = 0; col < fcsInfo.Columns; col++)
{
if (fcsInfo.ColumnInfo[col].LogTrans)
{
for (int row = 0; row < fcsInfo.Rows; row++)
{
m[row][col] = M * fastLogicle.scale(Math.Min(maxVal, Math.Max(minVal, m[row][col])));
}
logScaled = true;
}
}
}
string msg = "Dataset imported from " + fileName + ". Version: " + fcsInfo.version;
if (compensated)
{
msg += ", compensated";
}
if (logScaled)
{
msg += ", log-scaled";
}
string dsName = dataTable.SaveAsDataset(shortName, msg);
app.Folder.OpenDataset(dsName);
INumberTable nTable = app.GetNumberTable();
List <IColumnSpec> csList = nTable.ColumnSpecList as List <IColumnSpec>;
int fsc = csList.FindIndex(cs => cs.Id == "FSC-A");
int ssc = csList.FindIndex(cs => cs.Id == "SSC-A");
if ((fsc >= 0) && (ssc >= 0))
{
var xy = app.New.XyPlot(nTable);
xy.Show();
xy.XAxisIndex = fsc;
xy.YAxisIndex = ssc;
xy.AutoScaling = true;
xy.Redraw();
xy.CaptureMap();
xy.Close();
app.Map.Name = "FSC/SSC";
}
else
{
// We just do create a simple PCA view.
IPcaView pca = nTable.ShowPcaView();
pca.ResetView();
pca.CaptureMap();
app.Map.Redraw();
pca.Close();
app.Map.Name = "PCA-All";
}
app.Map.GlyphType = glyphName;
app.Map.Redraw();
if (showResult)
{
app.New.SpectrumBand(nTable).Show();
}
fcsInfo = null;
return(true);
}
