public async Task <Sequence <T> > RunUntilFound(GestureClass target = null)
{
Target = target ?? Target;
Activate();
var foundSeqeuence = new Sequence <T>()
{
SourcePath = (await PeakFinder.FindBestSequenceAsync()).ToArray()
};
softStop = true;
return(Current.Analyze(foundSeqeuence).Result as Sequence <T>);
}
public static async Task <Classifier> FindBestClassifier(DataSet <DKS> Dataset, GestureClass currentGC = null)
{
var sw = new System.Diagnostics.Stopwatch();
Func <Classifier, DataSet <DKS>, Task <ClassifierMetrics> > assessmentFunc;
DataSet <DKS> d;
string gestureName;
if (currentGC != null)
{
// First, create a special Dataset which contains only two GC's... the one we're looking at, and "other".
d = new DataSet <DKS>();
d.AddClass(currentGC.className);
d.AddClass("Other");
foreach (var seq in Dataset.Samples)
{
var s = new Sequence <DKS>()
{
SourcePath = seq.SourcePath
};
if (seq.TrueClassIndex == currentGC.index)
{
s.TrueClassIndex = 0;
}
else
{
s.TrueClassIndex = 1;
}
//s.TrueClassName = Dataset.ActualGestureClasses[s.TrueClassIndex].className;
d.AddSequence(s, skipBitmap: true);
}
assessmentFunc = assessClassifier_singleGC;
gestureName = currentGC.className;
}
else
{
d = Dataset;
assessmentFunc = assessClassifier_multiGC;
gestureName = "full dataset";
}
Classifier c = null, bestC = null;
ClassifierMetrics bestOverallMetric = new ClassifierMetrics()
{
CrossEntropyLoss = 1000, WeightedAccuracy = -1
}; // Ridiculously worse than any conceivable real classifier's metrics.
var fExNames = Enumerable.Range(0, 14).Select(n => PULL + n.ToString());
var fExList = fExNames.Select <string, FeatureExtractor <DKS> >(name => new FeatureListExtractor(name)).ToHashSet();
fExList.UnionWith(FeatureListExtractor.AllExtractors.Values);
var fExTestedList = new List <FeatureExtractor <DKS> >();
bool addedDerivedExtractors = false, addedPullExtractors = false;
while (fExList.Count > 0)
{
// Pop the first entry in the extractor list - using While & Pop lets us add new extractors /inside/ the loop.
var extractor = fExList.First();
fExList.Remove(extractor);
fExTestedList.Add(extractor);
// Create a fresh classifier to build based on this extractor
c = (extractor is FeatureClusterExtractor) ? new ClusterClassifier() : new Classifier();
sw.Start();
var loss = c.CreateMachine(d, extractor);
sw.Stop();
var createTime = sw.Elapsed.TotalMilliseconds;
sw.Reset();
extractor.metrics = await assessmentFunc(c, d);
extractor.metrics.CrossEntropyLoss = loss;
extractor.metrics.TimeToCreate = createTime;
//Log.Debug("MachineLearning|SpecialTest", $"Created special classifier for {currentGC.className} with {extractor.Name} in " +
// $"{createTime:f1}ms; loss {loss:f3} / score {extractor.metrics.OverallScore():f2}. Assessed in {extractor.metrics.TimePerDatapoint:f1} ms/pt, with {extractor.metrics.WeightedAccuracy:f1}% accuracy.");
if (extractor.metrics > bestOverallMetric)
{
bestOverallMetric = extractor.metrics;
bestC = c;
if (extractor.metrics.OverallScore() > 80) // Worse than 80% accuracy => don't bother reporting it to me.
{
Log.Debug("MachineLearning|SpecialTest", $"New best classifier for {gestureName}: {extractor.Name} in " +
$"{createTime:f1}ms; loss {loss:f3} / accuracy score {extractor.metrics.OverallScore():f2}%. Assessed in " +
$"{extractor.metrics.TimePerDatapoint:f1} ms/pt, with {extractor.metrics.WeightedAccuracy:f1}% raw accuracy.");
}
}
if (fExList.Count == 0 && !addedPullExtractors)
{
addedPullExtractors = true;
var bestExtractors1 = fExTestedList
.Where(extr => extr.Dimensions > 1)
.Where(extr => !extr.Name.IsOneOf("LinAccelVec", "GravityVec", "GyroVec", "RotQuat")) // 'Cause their 'pull' axes are already part of the pull-all-fourteen basic set.
.OrderByDescending(extr => extr.metrics)
.Take(3);
foreach (var bestEx in bestExtractors1)
{
fExList.Add(new FeatureListExtractor(PULLX + bestEx.Name));
if (bestEx.Dimensions >= 2)
{
fExList.Add(new FeatureListExtractor(PULLY + bestEx.Name));
}
if (bestEx.Dimensions >= 3)
{
fExList.Add(new FeatureListExtractor(PULLZ + bestEx.Name));
}
if (bestEx.Dimensions == 4)
{
fExList.Add(new FeatureAxisSelector <DKS>(3, bestEx)
{
Name = "PULL_W_" + bestEx.Name
});
}
}
}
if (fExList.Count == 0 && !addedDerivedExtractors)
{
addedDerivedExtractors = true;
var bestExtractors2 = fExTestedList.OrderByDescending(extr => extr.metrics).Take(3).ToList();
//Log.Debug("MachineLearning|SpecialTest", $"\nFirst place goes to {bestExtractors[0].Name} with {bestExtractors[0].ExtractorScore:f2}, " +
// $"second to {bestExtractors[1].Name} with {bestExtractors[1].ExtractorScore:f2}, and third to {bestExtractors[2].Name} with " +
// $"{bestExtractors[2].ExtractorScore:f2}.\n ");
var be0 = bestExtractors2[0];
var be1 = bestExtractors2[1];
var be2 = bestExtractors2[2];
if (be0.Dimensions + be1.Dimensions < 5)
{
fExList.Add(new FeatureListExtractor(be0.Name, be1.Name));
}
if (be0.Dimensions + be2.Dimensions < 5)
{
fExList.Add(new FeatureListExtractor(be0.Name, be2.Name));
}
if (be1.Dimensions + be2.Dimensions < 5)
{
fExList.Add(new FeatureListExtractor(be1.Name, be2.Name));
}
//if (be0.Dimensions + be1.Dimensions + be2.Dimensions < 6)
fExList.Add(new FeatureListExtractor(be0.Name, be1.Name, be2.Name));
//fExList.Add(new FeatureListExtractor(INTEGRATE + bestExtractors[0].Name));
fExList.Add(new FeatureClusterExtractor(be0.Name, be1.Name));
fExList.Add(new FeatureClusterExtractor(be0.Name, be2.Name));
fExList.Add(new FeatureClusterExtractor(be1.Name, be2.Name));
fExList.Add(new FeatureClusterExtractor(be0.Name, be1.Name, be2.Name));
}
}
//var lastExtractors = fExTestedList.Skip(fExTestedList.Count - 5).ToList();
//Log.Debug("MachineLearning|SpecialTest", $"\nDerived: {lastExtractors[0].Name} with {lastExtractors[0].ExtractorScore:f2}, " +
// $"{lastExtractors[1].Name} with {lastExtractors[1].ExtractorScore:f2}, {lastExtractors[2].Name} with " +
// $"{lastExtractors[2].ExtractorScore:f2}, {lastExtractors[3].Name} with {lastExtractors[3].ExtractorScore:f2}, and " +
// $" {lastExtractors[4].Name} with {lastExtractors[4].ExtractorScore:f2}.\n ");
var bestExtractors = fExTestedList.OrderByDescending(extr => extr.metrics).Take(3).ToList();
Log.Debug("MachineLearning|SpecialTest", $"For {gestureName}:");
var labelStrings = new string[] { "First place", "Second place", "Third place" };
for (int i = 0; i < 3; i++)
{
var bestExtr = bestExtractors[i];
Log.Debug("MachineLearning|SpecialTest", $" {labelStrings[i]} goes to {bestExtr.Name}, with a score of {bestExtr.metrics.OverallScore():f2} " +
$"(accuracy {(bestExtr.metrics.WeightedAccuracy):f1}%, loss {bestExtr.metrics.CrossEntropyLoss:f2}, time {bestExtr.metrics.TimePerDatapoint:f1}).");
}
return(bestC);
}
public void ProvideCue(GestureClass gestureClass = null)
{
Current.SelectedGestureClass = gestureClass ?? Current.Dataset.Classes.GetRandom();
Speech.Say(Current.SelectedGestureClass.className, speakRate: 2.0);
Stopwatch.Start();
}
