private static List <Tuple <int, TimeSpan, int> > _MeasureTime(int conc,
string engine, IDataScorerTransform scorer, ITransformer transformer,
int N, int ncall, bool cacheScikit)
{
var args = new TextLoader.Arguments()
{
Separator = "tab",
HasHeader = true,
Column = new[]
{
new TextLoader.Column("Label", DataKind.BL, 0),
new TextLoader.Column("SentimentText", DataKind.Text, 1)
}
};
var testFilename = FileHelper.GetTestFile("wikipedia-detox-250-line-test.tsv");
var times = new List <Tuple <int, TimeSpan, int> >();
using (var env = EnvHelper.NewTestEnvironment(seed: 1, conc: conc))
{
// Take a couple examples out of the test data and run predictions on top.
var testLoader = TextLoader.ReadFile(env, args, new MultiFileSource(testFilename));
IDataView cache;
if (cacheScikit)
{
cache = new ExtendedCacheTransform(env, new ExtendedCacheTransform.Arguments(), testLoader);
}
else
{
cache = new CacheDataView(env, testLoader, new[] { 0, 1 });
}
var testData = cache.AsEnumerable <SentimentData>(env, false);
if (engine == "mlnet")
{
Console.WriteLine("engine={0} N={1} ncall={2} cacheScikit={3}", engine, N, ncall, cacheScikit);
var fct = transformer.MakePredictionFunction <SentimentData, SentimentPrediction>(env);
var sw = new Stopwatch();
for (int call = 1; call <= ncall; ++call)
{
sw.Reset();
sw.Start();
for (int i = 0; i < N; ++i)
{
foreach (var input in testData)
{
fct.Predict(input);
}
}
sw.Stop();
times.Add(new Tuple <int, TimeSpan, int>(N, sw.Elapsed, call));
}
}
else if (engine == "scikit")
{
Console.WriteLine("engine={0} N={1} ncall={2} cacheScikit={3}", engine, N, ncall, cacheScikit);
var model = new ValueMapperPredictionEngine <SentimentData>(env, scorer, conc: conc);
var output = new ValueMapperPredictionEngine <SentimentData> .PredictionTypeForBinaryClassification();
var sw = new Stopwatch();
for (int call = 1; call <= ncall; ++call)
{
sw.Reset();
sw.Start();
for (int i = 0; i < N; ++i)
{
foreach (var input in testData)
{
model.Predict(input, ref output);
}
}
sw.Stop();
times.Add(new Tuple <int, TimeSpan, int>(N, sw.Elapsed, call));
}
}
else
{
throw new NotImplementedException($"Unknown engine '{engine}'.");
}
}
return(times);
}
private List <Tuple <int, TimeSpan, int, float[]> > _MeasureTime(int conc,
string strategy, string engine, IDataScorerTransform scorer, ITransformer trscorer, int ncall)
{
var args = new TextLoader.Arguments()
{
Separator = "tab",
HasHeader = true,
Column = new[]
{
new TextLoader.Column("Label", DataKind.BL, 0),
new TextLoader.Column("SentimentText", DataKind.Text, 1)
}
};
var testFilename = FileHelper.GetTestFile("wikipedia-detox-250-line-test.tsv");
var times = new List <Tuple <int, TimeSpan, int, float[]> >();
using (var env = EnvHelper.NewTestEnvironment(seed: 1, conc: conc))
{
// Take a couple examples out of the test data and run predictions on top.
var testLoader = TextLoader.ReadFile(env, args, new MultiFileSource(testFilename));
IDataView cache;
if (strategy.Contains("extcache"))
{
cache = new ExtendedCacheTransform(env, new ExtendedCacheTransform.Arguments(), testLoader);
}
else
{
cache = new CacheDataView(env, testLoader, new[] { 0, 1 });
}
var testData = cache.AsEnumerable <SentimentData>(env, false);
var testDataArray = cache.AsEnumerable <SentimentData>(env, false).ToArray();
int N = 1;
if (engine == "mlnet")
{
var model = trscorer.MakePredictionFunction <SentimentData, SentimentPrediction>(env);
var sw = new Stopwatch();
for (int call = 1; call <= ncall; ++call)
{
sw.Reset();
var pred = new List <float>();
sw.Start();
for (int i = 0; i < N; ++i)
{
if (strategy.Contains("array"))
{
foreach (var input in testDataArray)
{
pred.Add(model.Predict(input).Score);
}
}
else
{
foreach (var input in testData)
{
pred.Add(model.Predict(input).Score);
}
}
}
sw.Stop();
times.Add(new Tuple <int, TimeSpan, int, float[]>(N, sw.Elapsed, call, pred.ToArray()));
}
}
else if (engine == "scikit")
{
string allSchema = SchemaHelper.ToString(scorer.Schema);
Assert.IsTrue(allSchema.Contains("PredictedLabel:Bool:4; Score:R4:5; Probability:R4:6"));
var model = new ValueMapperPredictionEngine <SentimentData>(env, scorer, conc: conc);
var output = new ValueMapperPredictionEngine <SentimentData> .PredictionTypeForBinaryClassification();
var sw = new Stopwatch();
for (int call = 1; call <= ncall; ++call)
{
var pred = new List <float>();
sw.Reset();
sw.Start();
for (int i = 0; i < N; ++i)
{
if (strategy.Contains("array"))
{
foreach (var input in testDataArray)
{
model.Predict(input, ref output);
pred.Add(output.Score);
}
}
else
{
foreach (var input in testData)
{
model.Predict(input, ref output);
pred.Add(output.Score);
}
}
}
sw.Stop();
times.Add(new Tuple <int, TimeSpan, int, float[]>(N, sw.Elapsed, call, pred.ToArray()));
}
}
else
{
throw new NotImplementedException($"Unknown engine '{engine}'.");
}
}
return(times);
}
