/// <summary>
/// Train a model on a single example,
/// </summary>
/// <typeparam name="TOutput"></typeparam>
/// <param name="trainerMaker"></param>
/// <param name="checker"></param>
private static void TrivialHelper <TOutput>(Func <ITrainerHost, ITrainer <Instances, IPredictorProducing <TOutput> > > trainerMaker, Action <TOutput, TOutput> checker)
{
// The following simple instance should result in a "trivial" predictor for binary classification, regression, and multiclass, I think.
ListInstances instances = new ListInstances();
instances.AddInst(new Float[] { (Float)0.0 }, (Float)0);
instances.CopyMetadata(null);
ITrainerHost host = new TrainHost(new Random(1), 0);
var trainer = trainerMaker(host);
trainer.Train(instances);
IPredictor <Instance, TOutput> predictor = (IPredictor <Instance, TOutput>)trainer.CreatePredictor();
IPredictor <Instance, TOutput> loadedPredictor = default(IPredictor <Instance, TOutput>);
using (Stream stream = new MemoryStream())
{
using (RepositoryWriter writer = RepositoryWriter.CreateNew(stream, false))
{
ModelSaveContext.SaveModel(writer, predictor, "foo");
writer.Commit();
}
stream.Position = 0;
using (RepositoryReader reader = RepositoryReader.Open(stream, false))
{
ModelLoadContext.LoadModel(out loadedPredictor, reader, "foo");
}
Assert.AreNotEqual(default(IPredictor <Instance, TOutput>), loadedPredictor, "did not load expected model");
}
TOutput result = predictor.Predict(instances[0]);
TOutput loadedResult = loadedPredictor.Predict(instances[0]);
checker(result, loadedResult);
}
private void listInstances_ColumnClick(object sender, ColumnClickEventArgs e)
{
// Determine if clicked column is already the column that is being sorted.
if (e.Column == lvwColumnSorter.ColumnToSort)
{
// Reverse the current sort direction for this column.
if (lvwColumnSorter.OrderOfSort == SortOrder.Ascending)
{
lvwColumnSorter.OrderOfSort = SortOrder.Descending;
}
else
{
lvwColumnSorter.OrderOfSort = SortOrder.Ascending;
}
}
else
{
// Set the column number that is to be sorted; default to ascending.
lvwColumnSorter.ColumnToSort = e.Column;
lvwColumnSorter.OrderOfSort = SortOrder.Ascending;
}
// Perform the sort with these new sort options.
ListInstances.Sort();
}
public void TestRandomBalancedFoldCreation()
{
ListInstances li = CreateInstancesWithNKeys(5);
var foldCreator = new CVFoldCreation();
var cmd = new TLCArguments();
cmd.numFolds = 5;
cmd.command = Command.CrossValidation;
cmd.stratifyInstances = true;
int[] folds = foldCreator.CreateFoldIndicesStratified(li, cmd, new Random(1));
int[] expectedIndices = { 1, 0, 3, 4, 2 };
for (int i = 0; i < folds.Length; i++)
{
Assert.Equal <int>(folds[i], expectedIndices[i]);
}
li = CreateInstancesWithNKeys(7);
folds = foldCreator.CreateFoldIndicesStratified(li, cmd, new Random(1));
expectedIndices = new int[] { 1, 0, 4, 1, 0, 2, 3 };
for (int i = 0; i < folds.Length; i++)
{
Assert.Equal <int>(folds[i], expectedIndices[i]);
}
li = CreateInstancesWithNKeys(10);
folds = foldCreator.CreateFoldIndicesStratified(li, cmd, new Random(1));
expectedIndices = new int[] { 2, 1, 0, 3, 2, 4, 0, 4, 3, 1 };
for (int i = 0; i < folds.Length; i++)
{
Assert.Equal <int>(folds[i], expectedIndices[i]);
}
Done();
}
private ListInstances CreateInstancesWithNKeys(int n)
{
var li = new ListInstances();
var v = new WritableVector(new Float[] { 1, 2, 3 });
for (int i = 0; i < n; i++)
{
li.Add(new Instance(v, 1, i.ToString()));
}
return(li);
}
[Ignore] // REVIEW: OVA no longer implements BulkPredict.
public void MulticlassExampleTest()
{
string dataFilename = GetDataPath(TestDatasets.msm.trainFilename);
///********* Training a model *******//
// assume data is in memory in matrix/vector form. Sparse format also supported.
Float[][] data;
Float[] labels;
// below just reads some actual data into these arrays
PredictionUtil.ReadInstancesAsArrays(dataFilename, out data, out labels);
// Create an Instances dataset.
ListInstances instances = new ListInstances();
for (int i = 0; i < data.Length; i++)
{
instances.AddInst(data[i], labels[i]);
}
instances.CopyMetadata(null);
// Create a predictor and specify some non-default settings
var args = new OVA.OldArguments();
PredictionUtil.ParseArguments(args, "p=FR ps=iter:20");
ITrainer <Instances, IPredictor <Instance, Float[]> > trainer = new OVA(args, new TrainHost(new Random(1), 0));
// Train a predictor
trainer.Train(instances);
var predictor = trainer.CreatePredictor();
///********* Several ways to save models. Only binary can be used to-reload in TLC. *******//
// Save the model in internal binary format that can be used for loading it.
string modelFilename = Path.GetTempFileName();
PredictorUtils.Save(modelFilename, predictor, instances, null);
// Save the model as a plain-text description
string modelFilenameText = Path.GetTempFileName();
PredictorUtils.SaveText(predictor, instances.Schema.FeatureNames, modelFilenameText);
// Save the model in Bing's INI format
string modelFilenameIni = Path.GetTempFileName();
PredictorUtils.SaveIni(predictor, instances.Schema.FeatureNames, modelFilenameIni);
///********* Loading and making predictions with a previously saved model *******//
// Note: there are several alternative ways to construct instances
// For example, see FactoryExampleTest below that demonstrates named-feature : value pairs.
// Load saved model
IDataModel dataModel;
IDataStats dataStats;
var pred = PredictorUtils.LoadPredictor <Float[]>(out dataModel, out dataStats, modelFilename);
// Get predictions for instances
Float[][] predictions = new Float[instances.Count][];
for (int i = 0; i < instances.Count; i++)
{
predictions[i] = pred.Predict(instances[i]);
}
// REVIEW: This looks like it wasn't doing what was expected - OVA didn't override
// BulkPredict, so this wasn't using FastRank's BulkPredict.
Float[][] bulkPredictions = ((IBulkPredictor <Instance, Instances, Float[], Float[][]>)pred).BulkPredict(instances);
Assert.Equal(predictions.Length, bulkPredictions.Length);
for (int i = 0; i < predictions.Length; i++)
{
Assert.Equal(predictions[i].Length, bulkPredictions[i].Length);
for (int j = 0; j < predictions[i].Length; j++)
{
Assert.Equal(predictions[i][j], bulkPredictions[i][j]);
}
}
//test new testers
{
var results = new MulticlassTester(new MulticlassTesterArguments()).Test(pred, instances);
// Get metric names and print them alongside numbers
for (int i = 0; i < results.Length; i++)
{
Log("{0,-30} {1}", results[i].Name, results[i].Value);
}
// sanity check vs. original predictor
var results2 = new MulticlassTester(new MulticlassTesterArguments()).Test(predictor, instances);
Assert.Equal(results.Length, results2.Length);
for (int i = 0; i < results.Length; i++)
{
Assert.Equal(results[i].Name, results2[i].Name);
Assert.Equal(results[i].Value, results2[i].Value);
}
}
File.Delete(modelFilename);
File.Delete(modelFilenameText);
File.Delete(modelFilenameIni);
Done();
}
public void SimpleExampleTest()
{
RunMTAThread(() =>
{
string dataFilename = GetDataPath(TestDatasets.msm.trainFilename);
///********* Training a model *******//
// assume data is in memory in matrix/vector form. Sparse format also supported.
Float[][] data;
Float[] labels;
// below just reads some actual data into these arrays
PredictionUtil.ReadInstancesAsArrays(dataFilename, out data, out labels);
// Create an Instances dataset.
ListInstances instances = new ListInstances();
for (int i = 0; i < data.Length; i++)
{
instances.AddInst(data[i], labels[i]);
}
instances.CopyMetadata(null);
// Create a predictor and specify some non-default settings
var sub = new SubComponent <ITrainer <Instances, IPredictor <Instance, Float> >, SignatureOldBinaryClassifierTrainer>(
"FastRank", "nl=5 lr =0.25 iter= 20");
var trainer = sub.CreateInstance(new TrainHost(new Random(1), 0));
// Train a predictor
trainer.Train(instances);
var predictor = trainer.CreatePredictor();
///********* Several ways to save models. Only binary can be used to-reload in TLC. *******//
// Save the model in internal binary format that can be used for loading it.
string modelFilename = Path.GetTempFileName();
PredictorUtils.Save(modelFilename, predictor, instances, null);
// Save the model as a plain-text description
string modelFilenameText = Path.GetTempFileName();
PredictorUtils.SaveText(predictor, instances.Schema.FeatureNames, modelFilenameText);
// Save the model in Bing's INI format
string modelFilenameIni = Path.GetTempFileName();
PredictorUtils.SaveIni(predictor, instances.Schema.FeatureNames, modelFilenameIni);
///********* Loading and making predictions with a previously saved model *******//
// Note: there are several alternative ways to construct instances
// For example, see FactoryExampleTest below that demonstrates named-feature : value pairs.
// Load saved model
IDataModel dataModel;
IDataStats dataStats;
var pred = PredictorUtils.LoadPredictor <Float>(out dataModel, out dataStats, modelFilename);
var dp = pred as IDistributionPredictor <Instance, Float, Float>;
// Get predictions for instances
Float[] probPredictions = new Float[instances.Count];
Float[] rawPredictions = new Float[instances.Count];
Float[] rawPredictions1 = new Float[instances.Count];
for (int i = 0; i < instances.Count; i++)
{
probPredictions[i] = dp.PredictDistribution(instances[i], out rawPredictions[i]);
rawPredictions1[i] = dp.Predict(new Instance(data[i]));
}
Float[] bulkPredictions = ((IBulkPredictor <Instance, Instances, Float, Float[]>)pred).BulkPredict(instances);
Assert.Equal(rawPredictions.Length, bulkPredictions.Length);
Assert.Equal(rawPredictions.Length, rawPredictions1.Length);
for (int i = 0; i < rawPredictions.Length; i++)
{
Assert.Equal(rawPredictions[i], bulkPredictions[i]);
}
for (int i = 0; i < rawPredictions.Length; i++)
{
Assert.Equal(rawPredictions[i], rawPredictions1[i]);
}
//test new testers
{
var results = new ClassifierTester(new ProbabilityPredictorTesterArguments()).Test(pred, instances);
// Get metric names and print them alongside numbers
for (int i = 0; i < results.Length; i++)
{
Log("{0,-30} {1}", results[i].Name, results[i].Value);
}
// sanity check vs. original predictor
var results2 = new ClassifierTester(new ProbabilityPredictorTesterArguments()).Test(predictor, instances);
Assert.Equal(results.Length, results2.Length);
for (int i = 0; i < results.Length; i++)
{
Assert.Equal(results[i].Name, results2[i].Name);
Assert.Equal(results[i].Value, results2[i].Value);
}
}
File.Delete(modelFilename);
File.Delete(modelFilenameText);
File.Delete(modelFilenameIni);
});
Done();
}
private void listInstances_MouseClick(object sender, MouseEventArgs e)
{
// main
// internal
// global
// crawler
// graph
// packets
// search
// operation
// console
// ---
// Disconnect
if (e.Button != MouseButtons.Right)
{
return;
}
ListInstanceItem item = ListInstances.GetItemAt(e.X, e.Y) as ListInstanceItem;
if (item == null)
{
return;
}
ContextMenu menu = new ContextMenu();
if (item.Core == null)
{
menu.MenuItems.Add(new MenuItem("Login", new EventHandler(Click_Connect)));
}
else
{
MenuItem global = null;
if (item.Instance.Context.Lookup != null)
{
global = new MenuItem("Lookup");
global.MenuItems.Add(new MenuItem("Crawler", new EventHandler(Click_GlobalCrawler)));
global.MenuItems.Add(new MenuItem("Graph", new EventHandler(Click_GlobalGraph)));
global.MenuItems.Add(new MenuItem("Packets", new EventHandler(Click_GlobalPackets)));
global.MenuItems.Add(new MenuItem("Search", new EventHandler(Click_GlobalSearch)));
}
MenuItem operation = new MenuItem("Organization");
operation.MenuItems.Add(new MenuItem("Crawler", new EventHandler(Click_OpCrawler)));
operation.MenuItems.Add(new MenuItem("Graph", new EventHandler(Click_OpGraph)));
operation.MenuItems.Add(new MenuItem("Packets", new EventHandler(Click_OpPackets)));
operation.MenuItems.Add(new MenuItem("Search", new EventHandler(Click_OpSearch)));
MenuItem firewall = new MenuItem("Firewall");
firewall.MenuItems.Add(new MenuItem("Open", new EventHandler(Click_FwOpen)));
firewall.MenuItems.Add(new MenuItem("NAT", new EventHandler(Click_FwNAT)));
firewall.MenuItems.Add(new MenuItem("Blocked", new EventHandler(Click_FwBlocked)));
menu.MenuItems.Add(new MenuItem("Main", new EventHandler(Click_Main)));
menu.MenuItems.Add(new MenuItem("Internal", new EventHandler(Click_Internal)));
menu.MenuItems.Add(new MenuItem("Bandwidth", new EventHandler(Click_Bandwidth)));
menu.MenuItems.Add(new MenuItem("Transfers", new EventHandler(Click_Transfers)));
if (global != null)
{
menu.MenuItems.Add(global);
}
menu.MenuItems.Add(operation);
menu.MenuItems.Add(firewall);
menu.MenuItems.Add(new MenuItem("Console", new EventHandler(Click_Console)));
menu.MenuItems.Add(new MenuItem("-"));
menu.MenuItems.Add(new MenuItem("Logout", new EventHandler(Click_Disconnect)));
}
menu.Show(ListInstances, e.Location);
}
