public void CanSuccessfullyRetrieveQuotedData()
{
string dataPath = GetDataPath("QuotingData.csv");
var loader = new Legacy.Data.TextLoader(dataPath).CreateFrom <QuoteInput>(useHeader: true, separator: ',', allowQuotedStrings: true, supportSparse: false);
var environment = new MLContext();
Experiment experiment = environment.CreateExperiment();
Legacy.ILearningPipelineDataStep output = loader.ApplyStep(null, experiment) as Legacy.ILearningPipelineDataStep;
experiment.Compile();
loader.SetInput(environment, experiment);
experiment.Run();
IDataView data = experiment.GetOutput(output.Data);
Assert.NotNull(data);
using (var cursor = data.GetRowCursor((a => true)))
{
var IDGetter = cursor.GetGetter <float>(0);
var TextGetter = cursor.GetGetter <ReadOnlyMemory <char> >(1);
Assert.True(cursor.MoveNext());
float ID = 0;
IDGetter(ref ID);
Assert.Equal(1, ID);
ReadOnlyMemory <char> Text = new ReadOnlyMemory <char>();
TextGetter(ref Text);
Assert.Equal("This text contains comma, within quotes.", Text.ToString());
Assert.True(cursor.MoveNext());
ID = 0;
IDGetter(ref ID);
Assert.Equal(2, ID);
Text = new ReadOnlyMemory <char>();
TextGetter(ref Text);
Assert.Equal("This text contains extra punctuations and special characters.;*<>?!@#$%^&*()_+=-{}|[]:;'", Text.ToString());
Assert.True(cursor.MoveNext());
ID = 0;
IDGetter(ref ID);
Assert.Equal(3, ID);
Text = new ReadOnlyMemory <char>();
TextGetter(ref Text);
Assert.Equal("This text has no quotes", Text.ToString());
Assert.False(cursor.MoveNext());
}
}
public void CanSuccessfullyApplyATransform()
{
var loader = new Legacy.Data.TextLoader("fakeFile.txt").CreateFrom <Input>();
var environment = new MLContext();
Experiment experiment = environment.CreateExperiment();
Legacy.ILearningPipelineDataStep output = loader.ApplyStep(null, experiment) as Legacy.ILearningPipelineDataStep;
Assert.NotNull(output.Data);
Assert.NotNull(output.Data.VarName);
Assert.Null(output.Model);
}
public void CanSuccessfullyTrimSpaces()
{
string dataPath = GetDataPath("TrimData.csv");
var loader = new Legacy.Data.TextLoader(dataPath).CreateFrom <QuoteInput>(useHeader: true, separator: ',', allowQuotedStrings: false, supportSparse: false, trimWhitespace: true);
var environment = new MLContext();
Experiment experiment = environment.CreateExperiment();
Legacy.ILearningPipelineDataStep output = loader.ApplyStep(null, experiment) as Legacy.ILearningPipelineDataStep;
experiment.Compile();
loader.SetInput(environment, experiment);
experiment.Run();
IDataView data = experiment.GetOutput(output.Data);
Assert.NotNull(data);
using (var cursor = data.GetRowCursor((a => true)))
{
var IDGetter = cursor.GetGetter <float>(0);
var TextGetter = cursor.GetGetter <ReadOnlyMemory <char> >(1);
Assert.True(cursor.MoveNext());
float ID = 0;
IDGetter(ref ID);
Assert.Equal(1, ID);
ReadOnlyMemory <char> Text = new ReadOnlyMemory <char>();
TextGetter(ref Text);
Assert.Equal("There is a space at the end", Text.ToString());
Assert.True(cursor.MoveNext());
ID = 0;
IDGetter(ref ID);
Assert.Equal(2, ID);
Text = new ReadOnlyMemory <char>();
TextGetter(ref Text);
Assert.Equal("There is no space at the end", Text.ToString());
Assert.False(cursor.MoveNext());
}
}
public void CanSuccessfullyApplyATransform()
{
var collection = CollectionDataSource.Create(new List <Input>()
{
new Input {
Number1 = 1, String1 = "1"
}
});
var environment = new MLContext();
Experiment experiment = environment.CreateExperiment();
Legacy.ILearningPipelineDataStep output = (Legacy.ILearningPipelineDataStep)collection.ApplyStep(null, experiment);
Assert.NotNull(output.Data);
Assert.NotNull(output.Data.VarName);
Assert.Null(output.Model);
}
public void CanSuccessfullyRetrieveSparseData()
{
string dataPath = GetDataPath("SparseData.txt");
var loader = new Legacy.Data.TextLoader(dataPath).CreateFrom <SparseInput>(useHeader: true, allowQuotedStrings: false, supportSparse: true);
var environment = new MLContext();
Experiment experiment = environment.CreateExperiment();
Legacy.ILearningPipelineDataStep output = loader.ApplyStep(null, experiment) as Legacy.ILearningPipelineDataStep;
experiment.Compile();
loader.SetInput(environment, experiment);
experiment.Run();
IDataView data = experiment.GetOutput(output.Data);
Assert.NotNull(data);
using (var cursor = data.GetRowCursor((a => true)))
{
var getters = new ValueGetter <float>[] {
cursor.GetGetter <float>(0),
cursor.GetGetter <float>(1),
cursor.GetGetter <float>(2),
cursor.GetGetter <float>(3),
cursor.GetGetter <float>(4)
};
Assert.True(cursor.MoveNext());
float[] targets = new float[] { 1, 2, 3, 4, 5 };
for (int i = 0; i < getters.Length; i++)
{
float value = 0;
getters[i](ref value);
Assert.Equal(targets[i], value);
}
Assert.True(cursor.MoveNext());
targets = new float[] { 0, 0, 0, 4, 5 };
for (int i = 0; i < getters.Length; i++)
{
float value = 0;
getters[i](ref value);
Assert.Equal(targets[i], value);
}
Assert.True(cursor.MoveNext());
targets = new float[] { 0, 2, 0, 0, 0 };
for (int i = 0; i < getters.Length; i++)
{
float value = 0;
getters[i](ref value);
Assert.Equal(targets[i], value);
}
Assert.False(cursor.MoveNext());
}
}
public void CanSuccessfullyEnumerated()
{
var collection = CollectionDataSource.Create(new List <Input>()
{
new Input {
Number1 = 1, String1 = "1"
},
new Input {
Number1 = 2, String1 = "2"
},
new Input {
Number1 = 3, String1 = "3"
}
});
var environment = new MLContext();
Experiment experiment = environment.CreateExperiment();
Legacy.ILearningPipelineDataStep output = collection.ApplyStep(null, experiment) as Legacy.ILearningPipelineDataStep;
experiment.Compile();
collection.SetInput(environment, experiment);
experiment.Run();
IDataView data = experiment.GetOutput(output.Data);
Assert.NotNull(data);
using (var cursor = data.GetRowCursor((a => true)))
{
var IDGetter = cursor.GetGetter <float>(0);
var TextGetter = cursor.GetGetter <ReadOnlyMemory <char> >(1);
Assert.True(cursor.MoveNext());
float ID = 0;
IDGetter(ref ID);
Assert.Equal(1, ID);
ReadOnlyMemory <char> Text = new ReadOnlyMemory <char>();
TextGetter(ref Text);
Assert.Equal("1", Text.ToString());
Assert.True(cursor.MoveNext());
ID = 0;
IDGetter(ref ID);
Assert.Equal(2, ID);
Text = new ReadOnlyMemory <char>();
TextGetter(ref Text);
Assert.Equal("2", Text.ToString());
Assert.True(cursor.MoveNext());
ID = 0;
IDGetter(ref ID);
Assert.Equal(3, ID);
Text = new ReadOnlyMemory <char>();
TextGetter(ref Text);
Assert.Equal("3", Text.ToString());
Assert.False(cursor.MoveNext());
}
}
