private static void PrintPreview(DataDebuggerPreview data)
{
foreach (var colInfo in data.ColumnView)
{
Console.Write("{0,-25}", colInfo.Column.Name);
}
Console.WriteLine();
foreach (var row in data.RowView)
{
foreach (var kvPair in row.Values)
{
if (kvPair.Key == "Pixels")
{
var pixels = ((VBuffer <float>)kvPair.Value).DenseValues().Take(5);
Console.Write("{0}...", string.Join(",", pixels));
}
else
{
Console.Write("{0,-25}", kvPair.Value);
}
}
Console.WriteLine();
}
}
// This method using 'DebuggerExtensions.Preview()' should only be used when debugging/developing, not for release/production trainings
public static void PeekDataViewInConsole(MLContext mlContext, IDataView dataView, IEstimator <ITransformer> pipeline, int numberOfRows = 4)
{
string msg = string.Format("Peek data in DataView: Showing {0} rows with the columns", numberOfRows.ToString());
ConsoleWriteHeader(msg);
//https://github.com/dotnet/machinelearning/blob/master/docs/code/MlNetCookBook.md#how-do-i-look-at-the-intermediate-data
ITransformer transformer = pipeline.Fit(dataView);
IDataView transformedData = transformer.Transform(dataView);
// 'transformedData' is a 'promise' of data, lazy-loading. call Preview
//and iterate through the returned collection from preview.
DataDebuggerPreview preViewTransformedData = transformedData.Preview(maxRows: numberOfRows);
foreach (DataDebuggerPreview.RowInfo row in preViewTransformedData.RowView)
{
KeyValuePair <string, object>[] ColumnCollection = row.Values;
string lineToPrint = "Row--> ";
foreach (KeyValuePair <string, object> column in ColumnCollection)
{
lineToPrint += $"| {column.Key}:{column.Value}";
}
Console.WriteLine(lineToPrint + "\n");
}
}
public static void SaveAsMd(this DataOperationsCatalog _, DataDebuggerPreview preview, TextWriter writer)
{
writer.WriteLine($"| {String.Join(" | ", preview.Schema.Select(s => s.Name))} |");
writer.WriteLine($"| {String.Join(" | ", preview.Schema.Select(s => "------"))} |");
foreach (var row in preview.RowView)
{
writer.WriteLine($" | {String.Join(" | ", row.Values.Select(x => x.Value.ToString()))} | ");
}
}
private Task AddPoints(object regressor, IDataView testDataView)
{
return(Task.Run(() =>
{
var prediction = ((IPredictMany)regressor).PredictMany(testDataView);
result = prediction.Cast <IScore>().ToArray();
testDataPreview = testDataView.Preview(100000);
}).ContinueWith(async a => await UpdatePlot(CancellationToken.None), TaskScheduler.FromCurrentSynchronizationContext()));
}
private static void PrintPreview(DataDebuggerPreview data)
{
foreach (var colInfo in data.ColumnView)
{
Console.Write("{0,-25}", colInfo.Column.Name);
}
Console.WriteLine();
foreach (var row in data.RowView)
{
foreach (var kvPair in row.Values)
{
Console.Write("{0,-25}", kvPair.Value);
}
Console.WriteLine();
}
}
/// <summary>
/// Helper method to write the machine learning pipeline to the console.
/// </summary>
/// <param name="preview">The data preview to write.</param>
/// <param name="numberOfRows">The maximum number of rows to write.</param>
public static void WritePreview(DataDebuggerPreview preview, int numberOfRows = 10)
{
// set up a console table
var table = new Table(
TableConfiguration.Unicode(),
(from c in preview.ColumnView select c.Column.Name).ToArray());
// fill the table with results
foreach (var row in preview.RowView)
{
table.AddRow((from c in row.Values
select c.Value is VBuffer <float>? "<vector>" : c.Value
).ToArray());
}
// write the table
Console.WriteLine(table.ToString());
}
public static void ShowDataViewInConsole(MLContext mlContext, IDataView dataView, int numberOfRows = 4)
{
string msg = string.Format("Show data in DataView: Showing {0} rows with the columns", numberOfRows.ToString());
ConsoleWriteHeader(msg);
DataDebuggerPreview preViewTransformedData = dataView.Preview(maxRows: numberOfRows);
foreach (DataDebuggerPreview.RowInfo row in preViewTransformedData.RowView)
{
KeyValuePair <string, object>[] ColumnCollection = row.Values;
string lineToPrint = "Row--> ";
foreach (KeyValuePair <string, object> column in ColumnCollection)
{
lineToPrint += $"| {column.Key}:{column.Value}";
}
Console.WriteLine(lineToPrint + "\n");
}
}
/// <summary>
/// Helper method to write a single column preview to the console.
/// </summary>
/// <param name="preview">The data preview to write.</param>
/// <param name="column">The name of the column to write.</param>
public static void WritePreviewColumn(DataDebuggerPreview preview, string column)
{
// set up a console table
var table = new Table(TableConfiguration.Unicode(), new ColumnHeader(column));
// fill the table with results
foreach (var row in preview.RowView)
{
foreach (var col in row.Values)
{
if (col.Key == column)
{
var vector = (VBuffer <float>)col.Value;
table.AddRow(string.Concat(vector.DenseValues()));
}
}
}
// write the table
Console.WriteLine(table.ToString());
}
/// <summary>
/// Helper method to write the machine learning pipeline to the console.
/// </summary>
/// <param name="preview">The data preview to write.</param>
public static void WritePreview(DataDebuggerPreview preview)
{
// set up a console table
var table = new Table(
TableConfiguration.Unicode(),
preview.ColumnView.Select(c => new ColumnHeader(c.Column.Name)).ToArray());
// fill the table with results
foreach (var row in preview.RowView)
{
table.AddRow((from c in row.Values
select c.Value is VBuffer <float>? "<vector>" : c.Value
).ToArray());
}
// write the table
Console.WriteLine(table.ToString());
// set up a console table
table = new Table(TableConfiguration.Unicode(), new ColumnHeader("Location"));
// fill the table with results
foreach (var row in preview.RowView)
{
foreach (var col in row.Values)
{
if (col.Key == "Location")
{
var vector = (VBuffer <float>)col.Value;
table.AddRow(string.Concat(vector.DenseValues()));
}
}
}
// write the table
Console.WriteLine(table.ToString());
}
public void TestIDataView()
{
IDataView dataView = MakeDataFrameWithAllColumnTypes(10, withNulls: false);
DataDebuggerPreview preview = dataView.Preview();
Assert.Equal(10, preview.RowView.Length);
Assert.Equal(15, preview.ColumnView.Length);
Assert.Equal("Byte", preview.ColumnView[0].Column.Name);
Assert.Equal((byte)0, preview.ColumnView[0].Values[0]);
Assert.Equal((byte)1, preview.ColumnView[0].Values[1]);
Assert.Equal("Decimal", preview.ColumnView[1].Column.Name);
Assert.Equal((double)0, preview.ColumnView[1].Values[0]);
Assert.Equal((double)1, preview.ColumnView[1].Values[1]);
Assert.Equal("Double", preview.ColumnView[2].Column.Name);
Assert.Equal((double)0, preview.ColumnView[2].Values[0]);
Assert.Equal((double)1, preview.ColumnView[2].Values[1]);
Assert.Equal("Float", preview.ColumnView[3].Column.Name);
Assert.Equal((float)0, preview.ColumnView[3].Values[0]);
Assert.Equal((float)1, preview.ColumnView[3].Values[1]);
Assert.Equal("Int", preview.ColumnView[4].Column.Name);
Assert.Equal((int)0, preview.ColumnView[4].Values[0]);
Assert.Equal((int)1, preview.ColumnView[4].Values[1]);
Assert.Equal("Long", preview.ColumnView[5].Column.Name);
Assert.Equal((long)0, preview.ColumnView[5].Values[0]);
Assert.Equal((long)1, preview.ColumnView[5].Values[1]);
Assert.Equal("Sbyte", preview.ColumnView[6].Column.Name);
Assert.Equal((sbyte)0, preview.ColumnView[6].Values[0]);
Assert.Equal((sbyte)1, preview.ColumnView[6].Values[1]);
Assert.Equal("Short", preview.ColumnView[7].Column.Name);
Assert.Equal((short)0, preview.ColumnView[7].Values[0]);
Assert.Equal((short)1, preview.ColumnView[7].Values[1]);
Assert.Equal("Uint", preview.ColumnView[8].Column.Name);
Assert.Equal((uint)0, preview.ColumnView[8].Values[0]);
Assert.Equal((uint)1, preview.ColumnView[8].Values[1]);
Assert.Equal("Ulong", preview.ColumnView[9].Column.Name);
Assert.Equal((ulong)0, preview.ColumnView[9].Values[0]);
Assert.Equal((ulong)1, preview.ColumnView[9].Values[1]);
Assert.Equal("Ushort", preview.ColumnView[10].Column.Name);
Assert.Equal((ushort)0, preview.ColumnView[10].Values[0]);
Assert.Equal((ushort)1, preview.ColumnView[10].Values[1]);
Assert.Equal("String", preview.ColumnView[11].Column.Name);
Assert.Equal("0".AsMemory(), preview.ColumnView[11].Values[0]);
Assert.Equal("1".AsMemory(), preview.ColumnView[11].Values[1]);
Assert.Equal("Char", preview.ColumnView[12].Column.Name);
Assert.Equal((ushort)65, preview.ColumnView[12].Values[0]);
Assert.Equal((ushort)66, preview.ColumnView[12].Values[1]);
Assert.Equal("Bool", preview.ColumnView[13].Column.Name);
Assert.Equal(true, preview.ColumnView[13].Values[0]);
Assert.Equal(false, preview.ColumnView[13].Values[1]);
Assert.Equal("ArrowString", preview.ColumnView[14].Column.Name);
Assert.Equal("foo".ToString(), preview.ColumnView[14].Values[0].ToString());
Assert.Equal("foo".ToString(), preview.ColumnView[14].Values[1].ToString());
}
public void TestIDataViewWithNulls()
{
int length = 10;
IDataView dataView = MakeDataFrameWithAllColumnTypes(length, withNulls: true);
DataDebuggerPreview preview = dataView.Preview();
Assert.Equal(length, preview.RowView.Length);
Assert.Equal(15, preview.ColumnView.Length);
Assert.Equal("Byte", preview.ColumnView[0].Column.Name);
Assert.Equal((byte)0, preview.ColumnView[0].Values[0]);
Assert.Equal((byte)1, preview.ColumnView[0].Values[1]);
Assert.Equal((byte)4, preview.ColumnView[0].Values[4]);
Assert.Equal((byte)0, preview.ColumnView[0].Values[5]); // null row
Assert.Equal((byte)6, preview.ColumnView[0].Values[6]);
Assert.Equal("Decimal", preview.ColumnView[1].Column.Name);
Assert.Equal((double)0, preview.ColumnView[1].Values[0]);
Assert.Equal((double)1, preview.ColumnView[1].Values[1]);
Assert.Equal((double)4, preview.ColumnView[1].Values[4]);
Assert.Equal(double.NaN, preview.ColumnView[1].Values[5]); // null row
Assert.Equal((double)6, preview.ColumnView[1].Values[6]);
Assert.Equal("Double", preview.ColumnView[2].Column.Name);
Assert.Equal((double)0, preview.ColumnView[2].Values[0]);
Assert.Equal((double)1, preview.ColumnView[2].Values[1]);
Assert.Equal((double)4, preview.ColumnView[2].Values[4]);
Assert.Equal(double.NaN, preview.ColumnView[2].Values[5]); // null row
Assert.Equal((double)6, preview.ColumnView[2].Values[6]);
Assert.Equal("Float", preview.ColumnView[3].Column.Name);
Assert.Equal((float)0, preview.ColumnView[3].Values[0]);
Assert.Equal((float)1, preview.ColumnView[3].Values[1]);
Assert.Equal((float)4, preview.ColumnView[3].Values[4]);
Assert.Equal(float.NaN, preview.ColumnView[3].Values[5]); // null row
Assert.Equal((float)6, preview.ColumnView[3].Values[6]);
Assert.Equal("Int", preview.ColumnView[4].Column.Name);
Assert.Equal((int)0, preview.ColumnView[4].Values[0]);
Assert.Equal((int)1, preview.ColumnView[4].Values[1]);
Assert.Equal((int)4, preview.ColumnView[4].Values[4]);
Assert.Equal((int)0, preview.ColumnView[4].Values[5]); // null row
Assert.Equal((int)6, preview.ColumnView[4].Values[6]);
Assert.Equal("Long", preview.ColumnView[5].Column.Name);
Assert.Equal((long)0, preview.ColumnView[5].Values[0]);
Assert.Equal((long)1, preview.ColumnView[5].Values[1]);
Assert.Equal((long)4, preview.ColumnView[5].Values[4]);
Assert.Equal((long)0, preview.ColumnView[5].Values[5]); // null row
Assert.Equal((long)6, preview.ColumnView[5].Values[6]);
Assert.Equal("Sbyte", preview.ColumnView[6].Column.Name);
Assert.Equal((sbyte)0, preview.ColumnView[6].Values[0]);
Assert.Equal((sbyte)1, preview.ColumnView[6].Values[1]);
Assert.Equal((sbyte)4, preview.ColumnView[6].Values[4]);
Assert.Equal((sbyte)0, preview.ColumnView[6].Values[5]); // null row
Assert.Equal((sbyte)6, preview.ColumnView[6].Values[6]);
Assert.Equal("Short", preview.ColumnView[7].Column.Name);
Assert.Equal((short)0, preview.ColumnView[7].Values[0]);
Assert.Equal((short)1, preview.ColumnView[7].Values[1]);
Assert.Equal((short)4, preview.ColumnView[7].Values[4]);
Assert.Equal((short)0, preview.ColumnView[7].Values[5]); // null row
Assert.Equal((short)6, preview.ColumnView[7].Values[6]);
Assert.Equal("Uint", preview.ColumnView[8].Column.Name);
Assert.Equal((uint)0, preview.ColumnView[8].Values[0]);
Assert.Equal((uint)1, preview.ColumnView[8].Values[1]);
Assert.Equal((uint)4, preview.ColumnView[8].Values[4]);
Assert.Equal((uint)0, preview.ColumnView[8].Values[5]); // null row
Assert.Equal((uint)6, preview.ColumnView[8].Values[6]);
Assert.Equal("Ulong", preview.ColumnView[9].Column.Name);
Assert.Equal((ulong)0, preview.ColumnView[9].Values[0]);
Assert.Equal((ulong)1, preview.ColumnView[9].Values[1]);
Assert.Equal((ulong)4, preview.ColumnView[9].Values[4]);
Assert.Equal((ulong)0, preview.ColumnView[9].Values[5]); // null row
Assert.Equal((ulong)6, preview.ColumnView[9].Values[6]);
Assert.Equal("Ushort", preview.ColumnView[10].Column.Name);
Assert.Equal((ushort)0, preview.ColumnView[10].Values[0]);
Assert.Equal((ushort)1, preview.ColumnView[10].Values[1]);
Assert.Equal((ushort)4, preview.ColumnView[10].Values[4]);
Assert.Equal((ushort)0, preview.ColumnView[10].Values[5]); // null row
Assert.Equal((ushort)6, preview.ColumnView[10].Values[6]);
Assert.Equal("String", preview.ColumnView[11].Column.Name);
Assert.Equal("0", preview.ColumnView[11].Values[0].ToString());
Assert.Equal("1", preview.ColumnView[11].Values[1].ToString());
Assert.Equal("4", preview.ColumnView[11].Values[4].ToString());
Assert.Equal("", preview.ColumnView[11].Values[5].ToString()); // null row
Assert.Equal("6", preview.ColumnView[11].Values[6].ToString());
Assert.Equal("Char", preview.ColumnView[12].Column.Name);
Assert.Equal((ushort)65, preview.ColumnView[12].Values[0]);
Assert.Equal((ushort)66, preview.ColumnView[12].Values[1]);
Assert.Equal((ushort)69, preview.ColumnView[12].Values[4]);
Assert.Equal((ushort)0, preview.ColumnView[12].Values[5]); // null row
Assert.Equal((ushort)71, preview.ColumnView[12].Values[6]);
Assert.Equal("Bool", preview.ColumnView[13].Column.Name);
Assert.Equal(true, preview.ColumnView[13].Values[0]);
Assert.Equal(false, preview.ColumnView[13].Values[1]);
Assert.Equal(true, preview.ColumnView[13].Values[4]);
Assert.Equal(false, preview.ColumnView[13].Values[5]); // null row
Assert.Equal(true, preview.ColumnView[13].Values[6]);
Assert.Equal("ArrowString", preview.ColumnView[14].Column.Name);
Assert.Equal("foo", preview.ColumnView[14].Values[0].ToString());
Assert.Equal("foo", preview.ColumnView[14].Values[1].ToString());
Assert.Equal("foo", preview.ColumnView[14].Values[4].ToString());
Assert.Equal("", preview.ColumnView[14].Values[5].ToString()); // null row
Assert.Equal("foo", preview.ColumnView[14].Values[6].ToString());
}
