private static DataTable ToDataTableWithCols(this IDataMatrix dm, string[] columns)
{
var colIndexes = new int[columns.Length];
var table = new DataTable();
var cols = dm.Columns.Where(c => !c.Hidden).Select(c => c.Name.ToUpper()).ToList();
for (var i = 0; i < columns.Length; i++)
{
var name = columns[i].ToUpper();
var index = cols.FindIndex(c => c == name);
if (index < 0)
{
continue;
}
colIndexes[i] = index;
table.Columns.Add(dm.Columns[index].Name);
}
foreach (object[] row in dm)
{
object[] values = new object[colIndexes.Length];
for (int i = 0; i < colIndexes.Length; i++)
{
values[i] = row[colIndexes[i]];
}
table.Rows.Add(values);
}
return(table);
}
internal static IEnumerable <int> GetIndexes(this IDataMatrix dm, IEnumerable <string> columnNames)
{
foreach (var name in columnNames)
{
yield return(dm.Columns.First(c => c.Name.Equals(name, StringComparison.CurrentCultureIgnoreCase)).Index);
}
}
public static DataTable ToDataTable(this IDataMatrix dm, params string[] columns)
{
if (columns?.Length > 0)
{
return(dm.ToDataTableWithCols(columns));
}
var table = new DataTable();
for (var i = 0; i < dm.Columns.Count; i++)
{
if (dm.Columns[i].Hidden)
{
continue;
}
table.Columns.Add(dm.Columns[i].Name);
}
foreach (object[] row in dm)
{
object[] values = new object[dm.Columns.Count(c => !c.Hidden)];
for (int i = 0; i < values.Length; i++)
{
if (dm.Columns[i].Hidden)
{
continue;
}
values[i] = row[i];
}
table.Rows.Add(values);
}
return(table);
}
internal static IEnumerable <object> GetValues(this IDataMatrix dm, int rowIndex, IEnumerable <int> columnIndexes)
{
foreach (var index in columnIndexes)
{
yield return(dm[rowIndex][index]);
}
}
internal static void ComputeDependentCols(this IDataMatrix dm, int[] colIndexes)
{
//Compute dynamic columns
foreach (var colIndex in colIndexes)
{
var myCol = dm.Columns[colIndex];
if (myCol.DependsOn == null)
{
continue;
}
var dependsOn = GetIndexes(dm.Columns, myCol.DependsOn).ToArray();
if (myCol.Compute != null)
{
object args = null;
if (myCol.Initialize != null)
{
var colGroups = new object[dependsOn.Length][];
var k = 0;
foreach (var dependsOnIndex in dependsOn)
{
var colValues = new object[dm.RowCount];
for (var i = 0; i < dm.RowCount; i++)
{
colValues[i] = dm[i][dependsOnIndex];
}
colGroups[k++] = colValues;
}
args = myCol.Initialize(colGroups);
}
for (var i = 0; i < dm.RowCount; i++)
{
var values = new object[dependsOn.Length];
for (int j = 0; j < dependsOn.Length; j++)
{
values[j] = dm[i][dependsOn[j]];
}
dm[i][colIndex] = myCol.Compute(values, args);
}
}
}
}
public CLabelFeatureData(string[] featureNames, float[] labels, int[] groupId, IDataMatrix<float> feature)
: base(featureNames, labels, groupId)
{
this.feature = feature;
}
private IDataMatrix<ushort> EncodeFeatureValues(IDataMatrix<float> dataMatrix, CodeBook codeBook, int cThreads, bool fSparse)
{
//construct the matrix builder given if we want to have dense or sparse representations or not
IDataMatrixBuilderRam<ushort> dataMatrixBuilderRam;
if (fSparse)
{
dataMatrixBuilderRam = new DataMatrixBuilderRamSparse<ushort>(dataMatrix.NumRows, dataMatrix.NumCols, 0);
}
else
{
dataMatrixBuilderRam = new DataMatrixBuilderRamDense<ushort>(dataMatrix.NumRows, dataMatrix.NumCols);
}
MatrixEncoder matrixEncoder = new MatrixEncoder(dataMatrix, codeBook, dataMatrixBuilderRam);
ProcessorMT processorMT = new ProcessorMT(matrixEncoder, cThreads);
#if QUANTIZER_TIMER
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
#if VERBOSE
Console.WriteLine("Starting encoding timer...");
#endif
timer.Start();
#endif
processorMT.Process();
#if QUANTIZER_TIMER
timer.Stop();
#if VERBOSE
Console.WriteLine("Total encoding time: {0} seconds", 0.001 * timer.ElapsedMilliseconds);
#endif
#endif
return matrixEncoder.CodedMatrix;
}
private CodeBook ComputeCodeBook(IDataMatrix<float> dataMatrix, int cThreads)
{
//perform quantization
MatrixQuantizer matrixQuantizer = new MatrixQuantizer(dataMatrix);
ProcessorMT processorMT = new ProcessorMT(matrixQuantizer, cThreads);
#if QUANTIZER_TIMER
System.Diagnostics.Stopwatch timer = new System.Diagnostics.Stopwatch();
#if VERBOSE
Console.WriteLine("Starting codebook timer...");
#endif
timer.Start();
#endif
processorMT.Process();
#if QUANTIZER_TIMER
timer.Stop();
#if VERBOSE
Console.WriteLine("Total Codebook calculation time: {0} seconds", 0.001 * timer.ElapsedMilliseconds);
#endif
#endif
//code book
return matrixQuantizer.codeBook;
}
public EncoderThreadObj(CodeBook codeBook, IDataMatrix<float> dataMatrix, IDataMatrixBuilderRam<ushort> matrixBuilder)
{
//temporary data used by encoding one data group
this.cDataSize = dataMatrix.NumCols;
this.codedRow = new ushort[this.cDataSize];
//input data
this.dataMatrix = dataMatrix;
this.codeBook = codeBook;
//store the encoded features
this.matrixBuilder = matrixBuilder;
}
public QuantizerThreadObj(float[][] dataMap, IDataMatrix<float> dataMatrix)
{
//allocating temporary data used for quantization
this.cDataSize = dataMatrix.NumRows;
this.input = new float[this.cDataSize];
this.quantizedData = new ushort[this.cDataSize]; // This is not needed by CJCB's binning
this.sortedIndex = Vector.IndexArray(this.cDataSize);
this.changePointData = new float[this.cDataSize];
//this.bucketSize = new int[cDataSize];
//input data
this.dataMatrix = dataMatrix;
this.dataMap = dataMap;
}
public MatrixQuantizer(IDataMatrix<float> dataMatrix)
{
this.dataMatrix = dataMatrix;
this.dataMap = new float[this.dataMatrix.NumCols][];
}
private static void VerifyDeviation(IDataMatrix dm, int index, string name, int height, int deviation)
{
Assert.Equal(name, dm[index][0]);
Assert.Equal(height, dm[index][1]);
Assert.Equal(deviation, dm[index][2]);
}
public static IDataMatrix Pivot(this IDataMatrix dm, string dataField, AggregateFunction aggregate,
string[] rowFields, string[] columnFields)
{
var rowFieldIndexes = dm.GetIndexes(rowFields).ToArray();
var colFieldIndexes = dm.GetIndexes(columnFields).ToArray();
var dataFieldIndex = dm.Columns.First(c => c.Name.ToUpper() == dataField.ToUpper()).Index;
var items = new PivotEntry[dm.RowCount];
for (var i = 0; i < dm.RowCount; i++)
{
var entry = new PivotEntry
{
DataValue = dm[i][dataFieldIndex]
};
entry.SetRowValues(dm.GetValues(i, rowFieldIndexes).ToArray());
entry.SetColValues(dm.GetValues(i, colFieldIndexes).ToArray());
items[i] = entry;
}
var rows = (from p in items
group p by p.RowId into g
select new PivotRow
{
Row = g.Key,
Cols = g.GroupBy(c => c.ColId, c => c.DataValue, (key, c) => new PivotColumn {
Name = key, Values = c.ToArray()
}).ToArray()
}).ToArray();
var cols = new string[0];
foreach (var row in rows)
{
cols = cols.Union(row.Cols.Select(c => c.Name)).ToArray();
}
Array.Sort(cols);
var values = new object[rows.Length, cols.Length + 1];
for (var rowIndex = 0; rowIndex < values.GetLength(0); rowIndex++)
{
values[rowIndex, 0] = rows[rowIndex].Row;
for (var colIndex = 1; colIndex < values.GetLength(1); colIndex++)
{
values[rowIndex, colIndex] = GetCellValue(rows[rowIndex].Cols, cols[colIndex - 1], aggregate);
}
}
var columns = new string[cols.Length + 1];
columns[0] = string.Join("\v", rowFields);
for (var i = 0; i < cols.Length; i++)
{
columns[i + 1] = cols[i];
}
var result = DataMatrixFactory.Create(values, columns);
return(result);
}
public static IDataMatrix Pivot(this IDataMatrix dm, string rowField, string dataField, AggregateFunction aggregate, params string[] columnFields)
{
return(dm.Pivot(dataField, aggregate, new[] { rowField }, columnFields));
}
public CLabelFeatureData(IDataMatrix<float> feature)
: base(null, null, null)
{
this.feature = feature;
}
private static void VerifyStudents(IDataMatrix dm)
{
Assert.Equal("Johnson", dm[0][1]);
Assert.Equal(56m, dm[0][2]);
Assert.Equal("Williams", dm[1][1]);
}
public MatrixEncoder(IDataMatrix<float> dataMatrix, CodeBook codeBook, IDataMatrixBuilderRam<ushort> matrixBuilder)
{
this.dataMatix = dataMatrix;
this.codeBook = codeBook;
this.matrixBuilder = matrixBuilder;
}
