private static List <Tuple <int, TimeSpan, int> > _MeasureTime(int conc, string engine, IDataScorerTransform scorer, ITransformer transformer, int N, int ncall, bool cacheScikit) { var args = new TextLoader.Arguments() { Separator = "tab", HasHeader = true, Column = new[] { new TextLoader.Column("Label", DataKind.BL, 0), new TextLoader.Column("SentimentText", DataKind.Text, 1) } }; var testFilename = FileHelper.GetTestFile("wikipedia-detox-250-line-test.tsv"); var times = new List <Tuple <int, TimeSpan, int> >(); using (var env = EnvHelper.NewTestEnvironment(seed: 1, conc: conc)) { // Take a couple examples out of the test data and run predictions on top. var testLoader = TextLoader.ReadFile(env, args, new MultiFileSource(testFilename)); IDataView cache; if (cacheScikit) { cache = new ExtendedCacheTransform(env, new ExtendedCacheTransform.Arguments(), testLoader); } else { cache = new CacheDataView(env, testLoader, new[] { 0, 1 }); } var testData = cache.AsEnumerable <SentimentData>(env, false); if (engine == "mlnet") { Console.WriteLine("engine={0} N={1} ncall={2} cacheScikit={3}", engine, N, ncall, cacheScikit); var fct = transformer.MakePredictionFunction <SentimentData, SentimentPrediction>(env); var sw = new Stopwatch(); for (int call = 1; call <= ncall; ++call) { sw.Reset(); sw.Start(); for (int i = 0; i < N; ++i) { foreach (var input in testData) { fct.Predict(input); } } sw.Stop(); times.Add(new Tuple <int, TimeSpan, int>(N, sw.Elapsed, call)); } } else if (engine == "scikit") { Console.WriteLine("engine={0} N={1} ncall={2} cacheScikit={3}", engine, N, ncall, cacheScikit); var model = new ValueMapperPredictionEngine <SentimentData>(env, scorer, conc: conc); var output = new ValueMapperPredictionEngine <SentimentData> .PredictionTypeForBinaryClassification(); var sw = new Stopwatch(); for (int call = 1; call <= ncall; ++call) { sw.Reset(); sw.Start(); for (int i = 0; i < N; ++i) { foreach (var input in testData) { model.Predict(input, ref output); } } sw.Stop(); times.Add(new Tuple <int, TimeSpan, int>(N, sw.Elapsed, call)); } } else { throw new NotImplementedException($"Unknown engine '{engine}'."); } } return(times); }
private List <Tuple <int, TimeSpan, int, float[]> > _MeasureTime(int conc, string strategy, string engine, IDataScorerTransform scorer, ITransformer trscorer, int ncall) { var testFilename = FileHelper.GetTestFile("wikipedia-detox-250-line-test.tsv"); var times = new List <Tuple <int, TimeSpan, int, float[]> >(); /*using (*/ var env = EnvHelper.NewTestEnvironment(seed: 1, conc: conc); { if (engine == "mlnet") { var args = new TextLoader.Options() { Separators = new[] { '\t' }, HasHeader = true, Columns = new[] { new TextLoader.Column("Label", DataKind.Boolean, 0), new TextLoader.Column("SentimentText", DataKind.String, 1) } }; // Take a couple examples out of the test data and run predictions on top. var testLoader = new TextLoader(env, args).Load(new MultiFileSource(testFilename)); IDataView cache; if (strategy.Contains("extcache")) { cache = new ExtendedCacheTransform(env, new ExtendedCacheTransform.Arguments(), testLoader); } else { cache = new CacheDataView(env, testLoader, new[] { 0, 1 }); } //var testData = cache.AsEnumerable<SentimentDataBoolFloat>(env, false); //var testDataArray = cache.AsEnumerable<SentimentDataBoolFloat>(env, false).ToArray(); int N = 1; var model = ComponentCreation.CreatePredictionEngine <SentimentDataBoolFloat, SentimentPrediction>(env, trscorer); var sw = new Stopwatch(); for (int call = 1; call <= ncall; ++call) { sw.Reset(); var pred = new List <float>(); sw.Start(); for (int i = 0; i < N; ++i) { if (strategy.Contains("array")) { /* * foreach (var input in testDataArray) * pred.Add(model.Predict(input).Score); */ } else { /* * foreach (var input in testData) * pred.Add(model.Predict(input).Score); */ } } sw.Stop(); times.Add(new Tuple <int, TimeSpan, int, float[]>(N, sw.Elapsed, call, pred.ToArray())); } } else if (engine == "scikit") { var args = new TextLoader.Options() { Separators = new[] { '\t' }, HasHeader = true, Columns = new[] { new TextLoader.Column("Label", DataKind.Boolean, 0), new TextLoader.Column("SentimentText", DataKind.String, 1) } }; // Take a couple examples out of the test data and run predictions on top. var testLoader = new TextLoader(env, args).Load(new MultiFileSource(testFilename)); IDataView cache; if (strategy.Contains("extcache")) { cache = new ExtendedCacheTransform(env, new ExtendedCacheTransform.Arguments(), testLoader); } else { cache = new CacheDataView(env, testLoader, new[] { 0, 1 }); } //var testData = cache.AsEnumerable<SentimentDataBool>(env, false); //var testDataArray = cache.AsEnumerable<SentimentDataBool>(env, false).ToArray(); int N = 1; string allSchema = SchemaHelper.ToString(scorer.Schema); Assert.IsTrue(allSchema.Contains("PredictedLabel:Bool:4; Score:R4:5; Probability:R4:6")); var model = new ValueMapperPredictionEngine <SentimentDataBool>(env, scorer); var output = new ValueMapperPredictionEngine <SentimentDataBool> .PredictionTypeForBinaryClassification(); var sw = new Stopwatch(); for (int call = 1; call <= ncall; ++call) { var pred = new List <float>(); sw.Reset(); sw.Start(); for (int i = 0; i < N; ++i) { if (strategy.Contains("array")) { /* * foreach (var input in testDataArray) * { * model.Predict(input, ref output); * pred.Add(output.Score); * } */ } else { /* * foreach (var input in testData) * { * model.Predict(input, ref output); * pred.Add(output.Score); * } */ } } sw.Stop(); times.Add(new Tuple <int, TimeSpan, int, float[]>(N, sw.Elapsed, call, pred.ToArray())); } } else { throw new NotImplementedException($"Unknown engine '{engine}'."); } } return(times); }