public WaveletD4Transform(int dataDim) { dim = HaarTransform.RoundDimension(dataDim); InitializeConstants(); D4 = WaveTransforms.App.ScriptApp.New.NumberTable(dim, dim); // Algorithm adapated from: http://www.bearcave.com/misl/misl_tech/wavelets/daubechies/daub.java for (int row = 0; row < dim; row++) { double[] v = (D4.Matrix as double[][])[row]; Array.Clear(v, 0, v.Length); v[row] = 1.0; for (int n = v.Length; n >= 4; n >>= 1) { TransformD4(v, n); } } int interval = dim / 16; for (int k = 0; k < dim; k++) { D4.RowSpecList[k].Type = D4.ColumnSpecList[k].Group = (short)(k / interval); } List <int> rows = new List <int>(); for (int row = 0; row < dataDim; row++) { rows.Add(row); } D4 = D4.SelectRows(rows); }
/// <summary> /// Perform CCA analysis on columns of two number tables. /// </summary> /// <param name="tableX">A number table.</param> /// <param name="tableY">A number table.</param> /// <returns>A PlsResult structure.</returns> /// <remarks>This method finds directions for tableX and tableY; so that these two tables will /// have maximal correlation in those directions.</remarks> public PlsResult DoCCA(INumberTable tableX, INumberTable tableY) { double[][] X = (double[][])tableX.Matrix; double[][] Y = (double[][])tableY.Matrix; double[][] Cxy = Covariance(X, Y); double[][] Cyx = Covariance(Y, X); IMathAdaptor math = MultivariateAnalysisPlugin.App.GetMathAdaptor(); double[][] rCxx = math.InvertMatrix(Covariance(X, X)); double[][] rCyy = math.InvertMatrix(Covariance(Y, Y)); MakeSymmetric(rCxx); MakeSymmetric(rCyy); double[][] A = MatrixProduct(rCxx, Cxy); double[][] B = MatrixProduct(rCyy, Cyx); double[][] Cx = MatrixProduct(A, B); double[][] Cy = MatrixProduct(B, A); double[][] Wx, Wy; double[] Rx, Ry; math.EigenDecomposition(Cx, out Wx, out Rx); math.EigenDecomposition(Cy, out Wy, out Ry); // // Rx and Ry should be the equal upto permutation and dimension. // Wy can be calculated from Wx by Ry = sqrt(1/Rx) * B * Rx // /* * double x0 = Math.Sqrt(Rx[0]); * double x1 = Math.Sqrt(Rx[1]); * double x2 = Math.Sqrt(Rx[2]); * * double y0 = Math.Sqrt(Ry[0]); * double y1 = Math.Sqrt(Ry[1]); * double y2 = Math.Sqrt(Ry[2]); * * 0.7165 0.4906 0.2668 */ SortEigenVectors(Wx, Rx); SortEigenVectors(Wy, Ry); //ValidateMatrix(Wx); //ValidateMatrix(Wy); PlsResult ret = new PlsResult(); ret.ProjectionX = EigenProjection(tableX, Wx); ret.ProjectionY = EigenProjection(tableY, Wy); ret.EigenVectorsX = Wx; ret.EigenVectorsY = Wy; ret.EigenValuesX = Rx; ret.EigenValuesY = Ry; return(ret); }
/// <summary> /// Projects the rows of a data table to a set of eigenvectors. /// </summary> /// <param name="dataTable">The input data table.</param> /// <param name="eigenVectors">The eigenvectors as row vectors.</param> /// <returns>An INumberTable with row vectors as the projection.</returns> INumberTable EigenProjection(INumberTable dataTable, double[][] eigenVectors) { int rows = dataTable.Rows; int columns = dataTable.Columns; int outDim = eigenVectors.Length; INumberTable outTable = MultivariateAnalysisPlugin.App.ScriptApp.New.NumberTable(rows, outDim); double[][] outMatrix = (double[][])outTable.Matrix; double[][] m = (double[][])dataTable.Matrix; for (int row = 0; row < rows; row++) { for (int j = 0; j < outDim; j++) { double v = 0.0; for (int col = 0; col < columns; col++) { v += m[row][col] * eigenVectors[j][col]; } outMatrix[row][j] = v; } } for (int row = 0; row < rows; row++) { outTable.RowSpecList[row].CopyFrom(dataTable.RowSpecList[row]); } return(outTable); }
/// <summary> /// Perform PLS analysis on columns of two number tables. /// </summary> /// <param name="tableX">A number table.</param> /// <param name="tableY">A number table.</param> /// <returns>A PlsResult structure.</returns> /// <remarks>This method finds directions for tableX and tableY; so that these two tables will /// have maximal covariance in those directions.</remarks> public PlsResult DoPLS(INumberTable tableX, INumberTable tableY) { double[][] Cxy = Covariance((double[][])tableX.Matrix, (double[][])tableY.Matrix); double[][] Cyx = Covariance((double[][])tableY.Matrix, (double[][])tableX.Matrix); double[][] CxyCyx = MatrixProduct(Cxy, Cyx); double[][] CyxCxy = MatrixProduct(Cyx, Cxy); IMathAdaptor math = MultivariateAnalysisPlugin.App.GetMathAdaptor(); double[][] Wx, Wy; double[] Rx, Ry; // These two matrix might be slightly asymmetric due to calculation errors. MakeSymmetric(CxyCyx); MakeSymmetric(CyxCxy); math.EigenDecomposition(CxyCyx, out Wx, out Rx); math.EigenDecomposition(CyxCxy, out Wy, out Ry); SortEigenVectors(Wx, Rx); SortEigenVectors(Wy, Ry); PlsResult ret = new PlsResult(); ret.ProjectionX = EigenProjection(tableX, Wx); ret.ProjectionY = EigenProjection(tableY, Wy); ret.EigenVectorsX = Wx; ret.EigenVectorsY = Wy; ret.EigenValuesX = Rx; ret.EigenValuesY = Ry; return(ret); }
void AddCluster(INumberTable nt, double[] expLevel, int count, short type) { int offset = (nt.Tag == null) ? 0 : (int)(nt.Tag); nt.Tag = offset + count; double[][] m = nt.Matrix as double[][]; int columns = expLevel.Length; for (int col = 0; col < columns; col++) { double mean = expLevel[col]; if (mean > 0) { double[] v = NewGaussianSampling(count, expLevel[col]); for (int row = 0; row < count; row++) { m[offset + row][col] = v[row]; } } } for (int row = offset; row < (offset + count); row++) { nt.RowSpecList[row].Type = type; } }
public INumberTable Filter(INumberTable inTable, double lowFreq, double highFreq) { // Centralize the table. inTable = inTable.Clone(); IList <IList <double> > m = inTable.Matrix; for (int row = 0; row < inTable.Rows; row++) { for (int col = 0; col < inTable.Columns; col++) { m[row][col] -= columnMean[col]; } } INumberTable outTable = WalshTransform.Filter(inTable, lowFreq, highFreq, baseTable.Columns, baseTable); m = outTable.Matrix; for (int row = 0; row < outTable.Rows; row++) { for (int col = 0; col < outTable.Columns; col++) { m[row][col] += columnMean[col]; } } return(outTable); }
void ValidateMatrix(double[][] m) { INumberTable nt = MultivariateAnalysisPlugin.App.ScriptApp.New.NumberTable(m); INumberTable ntT = nt.Transpose2(); INumberTable I = nt.Multiply(ntT); I.ShowAsTable(); }
public FourierTransform(int dimension) { tDim = dimension; matrixReal = WaveTransforms.App.ScriptApp.New.NumberTable(tDim, tDim); matrixImg = WaveTransforms.App.ScriptApp.New.NumberTable(tDim, tDim); double[][] mr = matrixReal.Matrix as double[][]; double[][] mi = matrixImg.Matrix as double[][]; double w = -2 * Math.PI / dimension; double f = 1 / Math.Sqrt(tDim); IList <IRowSpec> rowListReal = matrixReal.RowSpecList; IList <IRowSpec> rowListImg = matrixImg.RowSpecList; for (int i = 0; i < tDim; i++) { for (int j = 0; j <= i; j++) { double wij = i * j * w; mr[i][j] = mr[j][i] = f * Math.Cos(wij); mi[i][j] = mi[j][i] = f * Math.Sin(wij); } short freq = (short)((i < tDim / 2) ? i : (tDim - i)); if (tDim % 2 == 0) { rowListReal[i].Type = freq; rowListImg[i].Type = freq; if (i == tDim / 2) { rowListImg[i].Type = 0; } } else { rowListReal[i].Type = freq; rowListImg[i].Type = freq; if (i == tDim / 2) { rowListImg[i].Type = rowListReal[i].Type = (short)(tDim / 2); } } matrixReal.ColumnSpecList[i].Group = rowListReal[i].Type; matrixImg.ColumnSpecList[i].Group = rowListImg[i].Type; } // indexes to select non-duplicate part of the fourier matrix. selectReal = new int[tDim / 2 + 1]; for (int i = 0; i < selectReal.Length; i++) { selectReal[i] = i; } selectImg = new int[(tDim % 2 == 0) ? (tDim / 2 - 1) : (tDim / 2)]; for (int i = 0; i < selectImg.Length; i++) { selectImg[i] = i + 1; } }
/// <summary> /// Multiply two matrix with padding and repeating if dimensions don't match. /// </summary> /// <param name="left">The left matrix.</param> /// <param name="right">The right matrix.</param> /// <returns>The product of two matrix.</returns> public static INumberTable MatrixProduct(INumberTable left, INumberTable right) { int dim = right.Rows; // The dimension of the transformation. int repeats = left.Columns / dim; if (left.Columns % dim > 0) { repeats++; } int outColumns = repeats * right.Columns; INumberTable prod = WaveTransforms.App.ScriptApp.New.NumberTable(left.Rows, outColumns); double[][] P = (double[][])prod.Matrix; double[][] L = (double[][])left.Matrix; double[][] R = (double[][])right.Matrix; for (int row = 0; row < prod.Rows; row++) { double[] Lrow = L[row]; for (int col = 0; col < prod.Columns; col++) { int col2 = col % right.Columns; int k0 = col / right.Columns * dim; double v = 0; for (int k = 0; k < dim; k++) { if ((k0 + k) < Lrow.Length) { v += Lrow[k0 + k] * R[k][col2]; } } P[row][col] = v; } } for (int row = 0; row < left.Rows; row++) { prod.RowSpecList[row].CopyFrom(left.RowSpecList[row]); } IList <IColumnSpec> pSpecList = prod.ColumnSpecList; IList <IColumnSpec> rSpecList = right.ColumnSpecList; for (int col = 0; col < prod.Columns; col++) { IColumnSpec cs = pSpecList[col]; cs.CopyFrom(rSpecList[col % right.Columns]); int n = col / right.Columns; if (n > 0) { cs.Id += "_" + n; } } return(prod); }
public void SetOperationMode() { operationMode = true; var app = DataModeling.App.ScriptApp; numTable = (app.SelectedItems.Count == 0) ? app.GetNumberTable() : app.GetSelectedNumberTable(); this.Text = "Operation Node List"; this.listView1.Columns[0].Text = "Name"; this.listView1.Columns[1].Text = "Type"; }
IList <int> ToIndexList(INumberTable nTable, bool attributeMode, IList <string> itemList) { if (attributeMode) { return(nTable.IndexOfColumns(itemList)); } else { return(nTable.IndexOfRows(itemList)); } }
public void SetExpression(INumberTable numTable, IList <IBody> bodyList, IList <IBody> orgBodies, int N) { NumTable = numTable; OrgBodies = orgBodies; BodyList = bodyList; this.N = N; row2bodyIdx = Enumerable.Range(0, N).Where(i => !OrgBodies[i].Disabled).ToArray(); col2bodyIdx = Enumerable.Range(N, OrgBodies.Count - N).Where(i => !OrgBodies[i].Disabled).ToArray(); MeanExpression = ExpThreshold(); }
public void Duplicate(INumberTable nTable, bool attributeMode, IList <string> itemList) { if (attributeMode) { nTable.AppendColumns(nTable.SelectColumnsById(itemList)); } else { nTable.Append(nTable.SelectRowsById(itemList)); } }
public INumberTable Transform(INumberTable inTable) { INumberTable rMatrix = matrixReal.SelectColumns(selectReal); INumberTable iMatrix = matrixImg.SelectColumns(selectImg); IList <IColumnSpec> cs = iMatrix.ColumnSpecList; for (int i = 0; i < cs.Count; i++) { cs[i].Id += "_i"; } return(MatrixProduct(inTable, rMatrix.AppendColumns(iMatrix))); }
public void Delete(INumberTable nTable, bool attributeMode, IList <string> itemList) { IList <int> idxList = ToIndexList(nTable, attributeMode, itemList); if (attributeMode) { nTable.RemoveColumns(idxList); } else { nTable.RemoveRows(idxList); } }
public IPcaView Show() { INumberTable nt = DataGenerator.App.ScriptApp.New.NumberTable(bodies.Count, 3); for (int row = 0; row < nt.Rows; row++) { nt.Matrix[row][0] = bodies[row].X; nt.Matrix[row][1] = bodies[row].Y; nt.Matrix[row][2] = bodies[row].Z; nt.RowSpecList[row].Type = (short)bodies[row].Type; } return(nt.ShowPcaView()); }
public void Scale(INumberTable nTable, bool attributeMode, IList <string> itemList, double factor) { double[][] m = (double[][])nTable.Matrix; IList <int> idxList = ToIndexList(nTable, attributeMode, itemList); if (factor == 0) { double maxValue = double.MinValue; foreach (int i in idxList) { if (attributeMode) { for (int row = 0; row < nTable.Rows; row++) { maxValue = Math.Max(maxValue, m[row][i]); } } else { for (int col = 0; col < nTable.Columns; col++) { maxValue = Math.Max(maxValue, m[i][col]); } } } if (maxValue != 0) { factor = Math.Abs(nTable.MaximumValue()) / Math.Abs(maxValue); } } foreach (int i in idxList) { if (attributeMode) { for (int row = 0; row < nTable.Rows; row++) { m[row][i] *= factor; } } else { for (int col = 0; col < nTable.Columns; col++) { m[i][col] *= factor; } } } }
public PcaTransform(INumberTable numberTable) { baseTable = numberTable.GetPcaEigenvectors(null, 0).Transpose(); columnMean = new double[numberTable.Columns]; IList <IList <double> > m = numberTable.Matrix; for (int col = 0; col < numberTable.Columns; col++) { columnMean[col] = 0; for (int row = 0; row < numberTable.Rows; row++) { columnMean[col] += m[row][col]; } columnMean[col] /= numberTable.Rows; } }
public static IPcaView Show(IList <Vector4D> points, int dimension) { INumberTable nt = DataGenerator.App.ScriptApp.New.NumberTable(points.Count, dimension); for (int row = 0; row < nt.Rows; row++) { nt.Matrix[row][0] = points[row].x; nt.Matrix[row][1] = points[row].y; if (dimension == 4) { nt.Matrix[row][2] = points[row].z; nt.Matrix[row][3] = points[row].w; } nt.RowSpecList[row].Type = (short)points[row].type; } return(nt.ShowPcaView()); }
public void Initialize(IDataset dataset, XmlElement filterNode) { INumberTable numTable = dataset.GetNumberTable(); int N = numTable.Rows; double[][] L = Matrix(N, N); int offset = numTable.Columns - N; for (int i = 0; i < N; i++) { for (int j = 0; j < N; j++) { if (i != j) { L[i][j] = -numTable.Matrix[i][offset + j]; } else { L[i][j] = 0; } } } for (int i = 0; i < N; i++) { double rowSum = 0; for (int j = 0; j < N; j++) { rowSum += L[i][j]; } L[i][i] = -rowSum; } IMathAdaptor math = GraphMetrics.App.GetMathAdaptor(); double[][] H = math.InvertMatrix(L); d = Matrix(N, N); for (int i = 0; i < N; i++) { for (int j = 0; j < N; j++) { d[i][j] = H[i][i] + H[j][j] - H[i][j] - H[j][i]; } } }
public HaarTransform(int dataDim) { dim = RoundDimension(dataDim); haar = WaveTransforms.App.ScriptApp.New.NumberTable(dim, dim); // Algorithm adapated from: http://www.cs.ucf.edu/~mali/haar/. double sqrt2 = 1 / Math.Sqrt(2.0); double[] vp = new double[dim]; for (int row = 0; row < dim; row++) { double[] v = (haar.Matrix as double[][])[row]; Array.Clear(v, 0, v.Length); v[row] = 1.0; Array.Clear(vp, 0, vp.Length); int w = dim; while (w > 1) { w /= 2; for (int i = 0; i < w; i++) { vp[i] = (v[2 * i] + v[2 * i + 1]) * sqrt2; vp[i + w] = (v[2 * i] - v[2 * i + 1]) * sqrt2; } Array.Copy(vp, v, 2 * w); } } int interval = dim / 16; for (int k = 0; k < dim; k++) { haar.RowSpecList[k].Type = haar.ColumnSpecList[k].Group = (short)(k / interval); } List <int> rows = new List <int>(); for (int row = 0; row < dataDim; row++) { rows.Add(row); } haar = haar.SelectRows(rows); }
public static INumberTable Filter(INumberTable inTable, double lowFreq, double highFreq, int dimension, INumberTable baseTable) { int lowIdx = (int)(lowFreq * dimension); int hiIdx = (int)(highFreq * dimension); List <int> columns = new List <int>(); for (int i = 0; i < dimension; i++) { if ((i >= lowIdx) && (i <= hiIdx)) { columns.Add(i); } } INumberTable B = baseTable.SelectColumns(columns); INumberTable Bt = B.Transpose2(); int b = B.Columns; int r = inTable.Rows; int m = inTable.Columns; INumberTable outTable = null; //Depending on the size of m relative to r and b, we can //optimize the calculation by arrange the matrix multiplication. if (2 * r * b < m * (r + b)) { // The complexity in this arrangement is 2*m*r*b. outTable = FourierTransform.MatrixProduct(FourierTransform.MatrixProduct(inTable, B), Bt); } else { // The complexity in this arrangement is m*m*(r+b). outTable = FourierTransform.MatrixProduct(inTable, FourierTransform.MatrixProduct(B, Bt)); } IList <IColumnSpec> oSpecList = outTable.ColumnSpecList; IList <IColumnSpec> iSpecList = inTable.ColumnSpecList; for (int col = 0; col < outTable.Columns; col++) { oSpecList[col].CopyFrom(iSpecList[col]); } return(outTable); }
void PreCalculate() { if (dataset == null) return; // Extract the relevant data table. var bs = dataset.BodyList; INumberTable nt = dataset.GetNumberTableView(); toIdx = Enumerable.Range(0, bs.Count).Where(i => !bs[i].Disabled).ToArray(); // Indexes of enabled bodies. int N = nt.Rows - pcaSamples; int[] enabledRows = toIdx.Where(i => i < N).ToArray(); if (enabledRows.Length == 0) throw new TException("No data available!"); P = new float[enabledRows.Length][]; MT.Loop(0, P.Length, row => { float[] R = P[row] = new float[nt.Columns]; double[] dsR = nt.Matrix[enabledRows[row]] as double[]; for (int col = 0; col < nt.Columns; col++) R[col] = (float)dsR[col]; }); // Reverse toIdx; int[] rIdx = Enumerable.Repeat(-1, bs.Count).ToArray(); for (int i = 0; i < toIdx.Length; i++) rIdx[toIdx[i]] = i; toIdx = rIdx; using (var gpu = new VisuMap.DxShader.GpuDevice()) dtP = DualAffinity.DotProduct(gpu, P, false); float[] singValues = new float[pcaMax]; float[][] PC = FastPca.DoFastReduction(P, pcaMax, singValues, true); P = VisuMap.MathUtil.NewMatrix<float>(PC.Length, pcaSamples); // P now links data points with the injected points on the main PCA axis. float span = 4.0f * singValues[0]; stepSize = span / (pcaSamples - 1); float x0 = - 0.5f * span; MT.ForEach(PC, (R, row) => { double yy = R.Skip(1).Sum(v => v * v); for (int col = 0; col < pcaSamples; col++) { double x = R[0] - (x0 + col * stepSize); P[row][col] = (float)Math.Sqrt(x * x + yy); } }); }
public void Normalize(INumberTable nTable, bool attributeMode, IList <string> itemList) { double[][] m = (double[][])nTable.Matrix; IList <int> idxList = ToIndexList(nTable, attributeMode, itemList); if (attributeMode) { foreach (int col in idxList) { double vMax = double.MinValue; double vMin = double.MaxValue; for (int row = 0; row < nTable.Rows; row++) { vMax = Math.Max(vMax, m[row][col]); vMin = Math.Min(vMin, m[row][col]); } double vRange = vMax - vMin; for (int row = 0; row < nTable.Rows; row++) { m[row][col] = (vRange == 0) ? 0 : (m[row][col] - vMin) / vRange; } } } else { foreach (int row in idxList) { double vMax = double.MinValue; double vMin = double.MaxValue; for (int col = 0; col < nTable.Columns; col++) { vMax = Math.Max(vMax, m[row][col]); vMin = Math.Min(vMin, m[row][col]); } double vRange = vMax - vMin; for (int col = 0; col < nTable.Columns; col++) { m[row][col] = (vRange == 0) ? 0 : (m[row][col] - vMin) / vRange; } } } }
public void Logarithmic(INumberTable nTable, bool attributeMode, IList <string> itemList) { double[][] m = (double[][])nTable.Matrix; foreach (int i in ToIndexList(nTable, attributeMode, itemList)) { if (attributeMode) { for (int row = 0; row < nTable.Rows; row++) { m[row][i] = Math.Log(1 + Math.Abs(m[row][i])); } } else { for (int col = 0; col < nTable.Columns; col++) { m[i][col] = Math.Log(1 + Math.Max(0, m[i][col])); } } } }
public void SetExpressionTable(INumberTable numTable, IForm featureView) { if ((featureView is IMapSnapshot) || (featureView is IMdsCluster)) { featureMap = featureView; } else { MsgBox.Alert("Only parnet window is not a map-snapshot or mds-cluster view."); return; } NumTable = numTable; OrgBodies = featureMap.BodyList; featureMap.GlyphSet = "Ordered Glyphs"; featureMap.Redraw(); minExpression = double.MaxValue; double maxExpression = double.MinValue; MT.ForEach(NumTable.Matrix, R => { double minV = double.MaxValue; double maxV = double.MinValue; foreach (var v in R) { minV = Math.Min(minV, v); maxV = Math.Max(maxV, v); } lock (this) { minExpression = Math.Min(minExpression, minV); maxExpression = Math.Max(maxExpression, maxV); } }); stepExpression = (maxExpression - minExpression) / 16; SingleCellPlugin.App.ItemsSelected += App_ItemsSelected; featureView.TheForm.FormClosing += (s, e) => { SingleCellPlugin.App.ItemsSelected -= App_ItemsSelected; }; }
public void Logicle(INumberTable nTable, bool attributeMode, IList <string> itemList) { double[][] m = (double[][])nTable.Matrix; double T = 262144; double W = 1.0; double M = 4.5; string[] settings = DataCleansing.App.ScriptApp.GetProperty( "DataCleansing.Logicle.Settings", "262144; 1.0; 4.5").Split(';'); if (settings.Length == 3) { double.TryParse(settings[0], out T); double.TryParse(settings[1], out W); double.TryParse(settings[2], out M); } FastLogicle fastLogicle = new FastLogicle(T, W, M); double maxVal = fastLogicle.MaxValue; double minVal = fastLogicle.MinValue; foreach (int i in ToIndexList(nTable, attributeMode, itemList)) { if (attributeMode) { for (int row = 0; row < nTable.Rows; row++) { m[row][i] = M * fastLogicle.scale(Math.Min(maxVal, Math.Max(minVal, m[row][i]))); } } else { for (int col = 0; col < nTable.Columns; col++) { m[i][col] = M * fastLogicle.scale(Math.Min(maxVal, Math.Max(minVal, m[i][col]))); } } } }
public PcaTransform NewPca(INumberTable numberTable) { return(new PcaTransform(numberTable)); }
public bool ImportFile(string fileName) { using (FileStream f = new FileStream(fileName, FileMode.Open, FileAccess.Read)) using (BinaryReader br = new BinaryReader(f)) { byte[] magic = br.ReadBytes(6); if (magic[0] != 0x93) { return(false); } byte[] version = br.ReadBytes(2); int headLen = 0; if (version[0] == 1) { headLen = br.ReadInt16(); } else { headLen = br.ReadInt32(); } byte[] bHead = br.ReadBytes(headLen); string sHead = Encoding.UTF8.GetString(bHead); sHead = sHead.Trim(new char[] { '{', ',', '}' }); string[] fs = sHead.Split(new char[] { ' ', ',', ':', '\'', ')', '(' }, StringSplitOptions.RemoveEmptyEntries); NumpyDtype dtype = NumpyDtype.DT_Unknown; bool isFortranOrder = false; List <int> dims = new List <int>(); for (int i = 0; i < fs.Length; i += 2) { switch (fs[i]) { case "descr": switch (fs[i + 1]) { case "<f4": dtype = NumpyDtype.DT_Float32; break; case "<f8": dtype = NumpyDtype.DT_Float64; break; case "<i4": dtype = NumpyDtype.DT_Int32; break; default: dtype = NumpyDtype.DT_Unknown; break; } break; case "fortran_order": isFortranOrder = (fs[i + 1] == "True"); break; case "shape": dims.Add(int.Parse(fs[i + 1])); for (int j = i + 2; j < fs.Length; j++) { if (char.IsDigit(fs[j][0])) { dims.Add(int.Parse(fs[j])); } } break; } } if (dtype == NumpyDtype.DT_Unknown) { MessageBox.Show("Only float32, float64 and int32 data types are supported!"); return(false); } if (dims.Count == 0) { MessageBox.Show("No shape defined!"); return(false); } var appNew = DataModeling.App.ScriptApp.New; if (dims.Count == 2) { INumberTable nt = null; if (isFortranOrder) { nt = appNew.NumberTable(dims[1], dims[0]); for (int row = 0; row < dims[1]; row++) { ReadRow(nt.Matrix[row] as double[], br, dtype); } nt.Transpose(); } else { nt = appNew.NumberTable(dims[0], dims[1]); for (int row = 0; row < dims[0]; row++) { ReadRow(nt.Matrix[row] as double[], br, dtype); } } if (DataModeling.App.ScriptApp.Dataset != null) { var bs = DataModeling.App.ScriptApp.Dataset.BodyListEnabled(); if (nt.Rows != bs.Count) { bs = DataModeling.App.ScriptApp.Map.SelectedBodies; if (nt.Rows != bs.Count) { bs = null; } } if (bs != null) { for (int i = 0; i < nt.Rows; i++) { nt.RowSpecList[i].CopyFromBody(bs[i]); } } } var hm = nt.ShowHeatMap(); hm.Title = "HeatMap: " + fileName; } else { int len = 1; foreach (int d in dims) { len *= d; } float[] values = new float[len]; for (int i = 0; i < len; i++) { values[i] = (float)NextValue(br, dtype); } var bb = appNew.BigBarView(values).Show(); bb.Title = "BigBar View: " + fileName; } } return(true); }
public bool ImportFile0(string fileName) { IVisuMap app = CustomImporter.App.ScriptApp; if (!fileName.ToLower().EndsWith(".fcs")) { return(false); // Let other import handle it. } INumberTable dataTable = null; IFreeTable textTable = null; bool compensated = false; bool logScaled = false; using (StreamReader sr = new StreamReader(fileName)) { // // Read the text segment. // char[] header = new char[42]; sr.ReadBlock(header, 0, header.Length); int textBegin = int.Parse(new string(header, 10, 8)); int textEnd = int.Parse(new string(header, 18, 8)); int beginData = int.Parse(new string(header, 26, 8)); int endData = int.Parse(new string(header, 34, 8)); char[] line = new char[textEnd + 4]; sr.ReadBlock(line, header.Length, line.Length - header.Length); string textSeg = new string(line, textBegin + 1, textEnd - textBegin); char delimiter = line[textBegin]; textTable = app.New.FreeTable(); textTable.AddColumn("Value", false); string[] textFields = textSeg.Split(delimiter); textTable.AddRows("P", textFields.Length / 2); IList <IRowSpec> rowSpecList = textTable.RowSpecList; for (int row = 0; row < textTable.Rows; row++) { rowSpecList[row].Id = textFields[2 * row]; textTable.Matrix[row][0] = textFields[2 * row + 1]; } // // Read in the data segement // fcsInfo = new FcsInfo(textTable, header); if ((beginData == 0) && (textTable.IndexOfRow("$BEGINDATA") > 0)) { beginData = int.Parse(textTable.Matrix[textTable.IndexOfRow("$BEGINDATA")][0]); } if ((endData == 0) && (textTable.IndexOfRow("$ENDDATA") > 0)) { endData = int.Parse(textTable.Matrix[textTable.IndexOfRow("$ENDDATA")][0]); } dataTable = app.New.NumberTable(fcsInfo.Rows, fcsInfo.Columns); using (BinaryReader br = new BinaryReader(sr.BaseStream)) { br.BaseStream.Seek(beginData, SeekOrigin.Begin); Byte[] buf4 = new byte[4]; Byte[] buf8 = new byte[8]; int bitOffset = 0; for (int row = 0; row < fcsInfo.Rows; row++) { for (int col = 0; col < fcsInfo.Columns; col++) { ColumnInfo ci = fcsInfo.ColumnInfo[col]; Byte[] data = ReadBits(br, ci.Bits, ref bitOffset); if (data.Length < ci.Bytes) { row = fcsInfo.Rows; // enforce premature loop-end. break; } Byte[] buf = (fcsInfo.DataType == FcsInfo.DataTypes.Double) ? buf8 : buf4; Array.Clear(buf, 0, buf.Length); Array.Copy(data, 0, buf, 0, ci.Bytes); if (!fcsInfo.BigEndian) { // Intel CPU expects BigEndian format. Array.Reverse(buf, 0, ci.Bytes); } switch (fcsInfo.DataType) { case FcsInfo.DataTypes.Integer: dataTable.Matrix[row][col] = (BitConverter.ToUInt32(buf, 0) & ci.RangeMask); break; case FcsInfo.DataTypes.Float: dataTable.Matrix[row][col] = BitConverter.ToSingle(buf, 0); break; case FcsInfo.DataTypes.Double: dataTable.Matrix[row][col] = BitConverter.ToDouble(buf, 0); break; } } } } } // Post processing for (int col = 0; col < fcsInfo.Columns; col++) { IColumnSpec cs = dataTable.ColumnSpecList[col]; ColumnInfo cInfo = fcsInfo.ColumnInfo[col]; cs.Id = cInfo.ShortName; //cs.Name = cInfo.Name + ( cInfo.IsLinear ? ".Lin" : ".Log"); cs.Name = cInfo.Name; if ((cs.Id == "TIME") || (cs.Id == "TIME1")) { int timeStepIdx = textTable.IndexOfRow("$TIMESTEP"); if (timeStepIdx >= 0) { double timeStep = double.Parse(textTable.Matrix[timeStepIdx][0]); for (int row = 0; row < fcsInfo.Rows; row++) { dataTable.Matrix[row][col] *= timeStep; } } } } IList <IRowSpec> rSpecList = dataTable.RowSpecList; for (int row = 0; row < fcsInfo.Rows; row++) { rSpecList[row].Id = row.ToString(); } FileInfo fInfo = new FileInfo(fileName); string shortName = fInfo.Name.Substring(0, fInfo.Name.LastIndexOf('.')); if (saveMetaInfo >= 1) { SaveParameterTable(textTable, shortName); if (saveMetaInfo >= 2) { textTable.SaveAsDataset(shortName + " (Text Seg)", "Text segement of data table " + shortName); } } if (autoCompensation) { try { string sMatrix = null; for (int i = 0; i < textTable.Rows; i++) { string id = textTable.RowSpecList[i].Id.ToLower(); if (id.StartsWith("$")) { id = id.Substring(1); } if ((id == "spill") || (id == "spillover")) { sMatrix = textTable.Matrix[i][0]; break; } } if (sMatrix == null) { throw new Exception(""); // silently ignore compensation as no compensation matrix available. } string[] fs = sMatrix.Split(','); int dimension = int.Parse(fs[0]); if (fs.Length != (dimension * dimension + dimension + 1)) { throw new Exception("Invalid spill over matrix."); } INumberTable cMatrix = app.New.NumberTable(dimension, dimension); List <string> parList = fs.Skip(1).Take(dimension).ToList(); int idx; if (parList.Count(id => int.TryParse(id, out idx)) == parList.Count) { // The columns are specified by a list of indexes. We convert them here to ids for (int i = 0; i < parList.Count; i++) { parList[i] = dataTable.ColumnSpecList[int.Parse(parList[i]) - 1].Id; // index starts with 1 ! } } for (int col = 0; col < cMatrix.Columns; col++) { cMatrix.ColumnSpecList[col].Id = parList[col]; } int fsIdx = dimension + 1; for (int row = 0; row < cMatrix.Rows; row++) { for (int col = 0; col < cMatrix.Columns; col++) { cMatrix.Matrix[row][col] = double.Parse(fs[fsIdx++]); } } var cData = dataTable.SelectColumnsById(parList); if (cData.Columns != parList.Count) { if (dataTable.ColumnSpecList.Count(cl => cl.Id.Equals("<" + parList[0] + ">")) == 1) { // siliently ignore aready compensated data. throw new Exception(""); } else { throw new Exception("Invalid spill over matrix: unknown names."); } } var math = CustomImporter.App.GetMathAdaptor(); var m = math.InvertMatrix((double[][])cMatrix.Matrix); for (int row = 0; row < cMatrix.Rows; row++) { cMatrix.Matrix[row] = m[row]; } cData = cData.Multiply(cMatrix); // perform the comensation with the inverse matrix of the spill over matrix. cData.CopyValuesTo(dataTable); compensated = true; } catch (Exception ex) { if (ex.Message != "") { MessageBox.Show("Cannot perform compensation: " + ex.Message); } } } if (logTransform) { double[][] m = (double[][])dataTable.Matrix; double T = 262144; double W = 1.0; double M = 4.5; string[] settings = app.GetProperty( "CustomImporter.Logicle.Settings", "262144; 1.0; 4.5").Split(';'); if (settings.Length == 3) { double.TryParse(settings[0], out T); double.TryParse(settings[1], out W); double.TryParse(settings[2], out M); } var fastLogicle = new VisuMap.DataCleansing.FastLogicle(T, W, M); double maxVal = fastLogicle.MaxValue; double minVal = fastLogicle.MinValue; for (int col = 0; col < fcsInfo.Columns; col++) { if (fcsInfo.ColumnInfo[col].LogTrans) { for (int row = 0; row < fcsInfo.Rows; row++) { m[row][col] = M * fastLogicle.scale(Math.Min(maxVal, Math.Max(minVal, m[row][col]))); } logScaled = true; } } } string msg = "Dataset imported from " + fileName + ". Version: " + fcsInfo.version; if (compensated) { msg += ", compensated"; } if (logScaled) { msg += ", log-scaled"; } string dsName = dataTable.SaveAsDataset(shortName, msg); app.Folder.OpenDataset(dsName); INumberTable nTable = app.GetNumberTable(); List <IColumnSpec> csList = nTable.ColumnSpecList as List <IColumnSpec>; int fsc = csList.FindIndex(cs => cs.Id == "FSC-A"); int ssc = csList.FindIndex(cs => cs.Id == "SSC-A"); if ((fsc >= 0) && (ssc >= 0)) { var xy = app.New.XyPlot(nTable); xy.Show(); xy.XAxisIndex = fsc; xy.YAxisIndex = ssc; xy.AutoScaling = true; xy.Redraw(); xy.CaptureMap(); xy.Close(); app.Map.Name = "FSC/SSC"; } else { // We just do create a simple PCA view. IPcaView pca = nTable.ShowPcaView(); pca.ResetView(); pca.CaptureMap(); app.Map.Redraw(); pca.Close(); app.Map.Name = "PCA-All"; } app.Map.GlyphType = glyphName; app.Map.Redraw(); if (showResult) { app.New.SpectrumBand(nTable).Show(); } fcsInfo = null; return(true); }