private async void MyTransformButton_Click(object sender, RoutedEventArgs e)
{
int resample = int.Parse(MyResampleText.Text);
double scale = double.Parse(MyScaleText.Text);
double x = double.Parse(MyTranslateX.Text);
double y = double.Parse(MyTranslateY.Text);
Point point = new Point(x, y);
foreach (StorageFile loadFile in myLoadFiles)
{
string loadFileName = loadFile.Name;
Sketch sketch = await SketchTools.XmlToSketch(loadFile);
if (MyResampleCheckBox.IsChecked.Value)
{
sketch = SketchTransformation.Resample(sketch, resample);
}
if (MyScaleCheckBox.IsChecked.Value)
{
sketch = SketchTransformation.ScaleFrame(sketch, scale);
}
if (MyTranslateCheckBox.IsChecked.Value)
{
sketch = SketchTransformation.TranslateFrame(sketch, point);
}
StorageFile saveFile = await mySaveFolder.CreateFileAsync(loadFileName, CreationCollisionOption.ReplaceExisting);
SketchTools.SketchToXml(saveFile, sketch.Label, sketch.Strokes, sketch.Times, sketch.FrameMinX, sketch.FrameMinY, sketch.FrameMaxX, sketch.FrameMaxY);
}
}
private void MyStrokeBoundsPlayButton_Click(object sender, RoutedEventArgs e)
{
// get the model and input strokes
Sketch model = SketchTools.Clone(myTemplates[MyCurrentIndex]);
Sketch input = Sketch.CreateStroke("", new List <InkStroke>(MyInkStrokes.GetStrokes()), myTimeCollection, 0, 0, BorderLength, BorderLength);
// TODO
}
private void MyPage_Loaded(object sender, RoutedEventArgs e)
{
// initializer the feedback recognizers
myStructureRecognizer = new StructureRecognizer();
myTechniqueRecognizer = new TechniqueRecognizer();
// squarify the ink canvas' drawing area
double width = MyBorder.ActualWidth;
double height = MyBorder.ActualHeight;
BorderLength = width < height ? width : height;
MyBorder.Width = MyBorder.Height = BorderLength;
// initialize the external timing data structure and offset
myTimeCollection = new List <List <long> >();
MyDateTimeOffset = 0;
// enable writing and set the stroke visuals of the ink canvas
MyInkStrokes = MyInkCanvas.InkPresenter.StrokeContainer;
MyInkCanvas.InkPresenter.InputDeviceTypes = CoreInputDeviceTypes.Pen | CoreInputDeviceTypes.Mouse | CoreInputDeviceTypes.Touch;
MyInkCanvas.InkPresenter.UpdateDefaultDrawingAttributes(StrokeVisuals);
// load the images and templates
LoadContents(IMAGES_PATH, out myImageFiles, ".png");
LoadContents(TEMPLATES_PATH, out myTemplateFiles, ".xml");
// populate the symbols combo box
foreach (StorageFile file in myImageFiles)
{
MySymbolsComboBox.Items.Add(Path.GetFileNameWithoutExtension(file.Path));
}
MySymbolsComboBox.SelectedIndex = 0;
// set the prompt text
string promptedSymbol = MySymbolsComboBox.SelectedValue.ToString().ToUpper();
MyPrompText.Text = PROMPT_TEXT + promptedSymbol;
// modify the templates to fit the current ink canvas' drawing area
myTemplates = new List <Sketch>();
foreach (StorageFile file in myTemplateFiles)
{
Sketch template = null;
Task task = Task.Run(async() => template = await SketchTools.XmlToSketch(file));
task.Wait();
template = SketchTransformation.ScaleFrame(template, BorderLength);
template = SketchTransformation.TranslateFrame(template, new Point(BorderLength / 2 - MyBorder.BorderThickness.Left, BorderLength / 2 - MyBorder.BorderThickness.Top));
myTemplates.Add(template);
foreach (InkStroke stroke in template.Strokes)
{
stroke.DrawingAttributes = StrokeVisuals;
}
}
// set the flag to allow buttons to run code
MyIsReady = true;
}
private async void MySymbolCorrectnessPlayButton_Click(object sender, RoutedEventArgs e)
{
//
//EnableStructureButtons(false);
MyImage.Visibility = Visibility.Collapsed;
// get the model and input strokes
Sketch model = SketchTools.Clone(myTemplates[MyCurrentIndex]);
Sketch input = Sketch.CreateStroke("", new List <InkStroke>(MyInkStrokes.GetStrokes()), myTimeCollection, 0, 0, BorderLength, BorderLength);
// store and remove the original strokes from the ink canvas
List <InkStroke> originalStrokes = SketchTools.Clone(new List <InkStroke>(MyInkStrokes.GetStrokes()));
foreach (InkStroke stroke in MyInkStrokes.GetStrokes())
{
stroke.Selected = true;
}
MyInkStrokes.DeleteSelected();
// set the animation colors
SolidColorBrush modelBrush = new SolidColorBrush(Colors.Green)
{
Opacity = 1.0
};
SolidColorBrush inputBrush = new SolidColorBrush(Colors.Red)
{
Opacity = 1.0
};
// get the animations
List <Storyboard> modelStoryboards = Helper.DisplaySymbol(MyCanvas, model.Strokes, modelBrush, SMALL_DOT_SIZE, STROKE_DURATION);
List <Storyboard> inputStoryboards = Helper.DisplaySymbol(MyCanvas, input.Strokes, inputBrush, StrokeVisuals.Size.Width, STROKE_DURATION);
// animate the feedback
foreach (Storyboard storyboard in modelStoryboards)
{
storyboard.Begin();
}
foreach (Storyboard storyboard in inputStoryboards)
{
storyboard.Begin();
}
// re-add the original strokes to the ink canvas and re-enable return button
int delay = STROKE_DURATION;
await InteractionTools.Delay(delay);
MyInkStrokes.AddStrokes(originalStrokes);
//
//EnableStructureButtons(true);
MyImage.Visibility = Visibility.Visible;
}
public void Train(Sketch model, List <Sketch> templates, Sketch input)
{
// set the model and input sketches
myModel = SketchTools.Clone(model);
myInput = SketchTools.Clone(input);
// set the template sketches
myTemplates = new List <Sketch>();
foreach (Sketch template in templates)
{
myTemplates.Add(SketchTools.Clone(template));
}
}
private async void MyStrokeCountPlayButton_Click(object sender, RoutedEventArgs e)
{
// do not show feedback if canvas has no strokes
if (MyInkStrokes.GetStrokes().Count <= 0)
{
return;
}
//
EnableTechniqueButtons(false);
// get the model and input strokes
Sketch model = myTemplates[MyCurrentIndex];
model = SketchTools.Clone(model);
Sketch input = Sketch.CreateStroke("", new List <InkStroke>(MyInkStrokes.GetStrokes()), myTimeCollection, 0, 0, BorderLength, BorderLength);
input = SketchTools.Clone(input);
input = SketchTransformation.Resample(input, 128);
// set the animation colors
SolidColorBrush modelBrush = new SolidColorBrush(Colors.Green)
{
Opacity = 1.0
};
SolidColorBrush inputBrush = new SolidColorBrush(Colors.Red)
{
Opacity = 1.0
};
// get the animations
List <Storyboard> modelStoryboards = InteractionTools.DisplayEndpoints(MyCanvas, model.Strokes, modelBrush, LARGE_DOT_SIZE, STROKE_DURATION);
List <Storyboard> inputStoryboards = InteractionTools.DisplayEndpoints(MyCanvas, input.Strokes, inputBrush, SMALL_DOT_SIZE, STROKE_DURATION, model);
// animate the feedback
foreach (Storyboard storyboard in modelStoryboards)
{
storyboard.Begin();
}
foreach (Storyboard storyboard in inputStoryboards)
{
storyboard.Begin();
}
// re-add the original strokes to the ink canvas and re-enable return button
int delay = STROKE_DURATION * model.Strokes.Count;
await InteractionTools.Delay(delay);
//
EnableTechniqueButtons(true);
}
private List <Tuple <InkStroke, InkStroke> > GetStrokeMatches(Sketch input, Sketch model)
{
// get the cloned input and model sketches;
// necessary for not affecting use of input and model sketches in other methods
input = SketchTools.Clone(input);
model = SketchTools.Clone(model);
// get the matching input and model strokes
var pairs = new List <Tuple <InkStroke, InkStroke> >();
for (int i = 0; i < input.Strokes.Count; ++i)
{
// initialize the minimal values
double minDistance = double.MaxValue;
int minIndex = -1;
// iterate through each model stroke
for (int j = 0; j < model.Strokes.Count; ++j)
{
// get the current model stroke
InkStroke modelStroke = model.Strokes[j];
// get the pairwise distance of the input and model
double distance = GetResampledPairwiseDistance(input, model, i, j);
// case: minimum distance found
// set the minimum distance and index
if (distance < minDistance)
{
minDistance = distance;
minIndex = j;
}
}
// set the matching input and model strokes
InkStroke inputStroke = input.Strokes[i];
InkStroke minModelStroke = model.Strokes[minIndex];
pairs.Add(new Tuple <InkStroke, InkStroke>(inputStroke, minModelStroke));
// remove the current model stroke and times candidate for the next iteration
model.Strokes.RemoveAt(minIndex);
model.Times.RemoveAt(minIndex);
}
return(pairs);
}
private async void MyPlayButton_Click(object sender, RoutedEventArgs e)
{
//
if (!MyImageButton.IsChecked.Value)
{
return;
}
//
EnableAppBarButtons(false);
MyInkCanvas.InkPresenter.IsInputEnabled = false;
//
Sketch model = myTemplates[MyCurrentIndex];
model = SketchTools.Clone(model);
double opacity = 0.8;
Color color = Colors.Green;
SolidColorBrush brush = new SolidColorBrush(color)
{
Opacity = opacity
};
// animate the expert's model strokes
List <Storyboard> modelStoryboards = InteractionTools.Trace(MyCanvas, model.Strokes, model.Times, LARGE_DOT_SIZE, brush, POINT_DURATION);
foreach (Storyboard storyboard in modelStoryboards)
{
storyboard.Begin();
}
//
int numModelPoints = 0;
foreach (InkStroke stroke in model.Strokes)
{
numModelPoints += stroke.GetInkPoints().Count;
}
int delay = numModelPoints * POINT_DURATION;
await InteractionTools.Delay(delay);
//
EnableAppBarButtons(true);
MyInkCanvas.InkPresenter.IsInputEnabled = true;
}
private double GetResampledPairwiseDistance(Sketch input, Sketch model, int i, int j)
{
// get the corresponding model stroke, points, and point count
InkStroke modelStroke = model.Strokes[j];
List <InkPoint> modelPoints = new List <InkPoint>(modelStroke.GetInkPoints());
int numModelPoints = modelPoints.Count;
// wrap the current input stroke into a sketch, then resample to match model stroke
InkStroke inputStroke = input.Strokes[i];
List <long> inputTimes = input.Times[i];
Sketch inputSketch = new Sketch("", new List <InkStroke>()
{
inputStroke
}, new List <List <long> > {
inputTimes
}, 0, 0, 0, 0);
// resample the input stroke's points to match the current model stroke's points
inputSketch = SketchTools.Clone(inputSketch);
inputSketch = SketchTransformation.Resample(inputSketch, modelPoints.Count);
inputStroke = inputSketch.Strokes[0];
// get the number of points to iterate between the corresponding model and input stroke
int numInputPoints = inputStroke.GetInkPoints().Count;
int count = numModelPoints < numInputPoints ? numModelPoints : numInputPoints;
// calculate the distances both forward and backwards between the model and input strokes
double distance1 = 0.0;
double distance2 = 0.0;
for (int a = 0, b = count - 1; a < count; ++a, --b)
{
InkPoint modelPoint = modelStroke.GetInkPoints()[a];
InkPoint inputPoint1 = inputStroke.GetInkPoints()[a];
InkPoint inputPoint2 = inputStroke.GetInkPoints()[b];
distance1 += SketchTransformation.Distance(modelPoint, inputPoint1);
distance2 += SketchTransformation.Distance(modelPoint, inputPoint2);
}
double distance = distance1 < distance2 ? distance1 : distance2;
return(distance);
}
private bool SymbolCorrectnessTest(Sketch model, List <Sketch> templates, Sketch input)
{
PDollar dollar = new PDollar(128, 500, new Point(0, 0));
dollar.Train(templates);
dollar.Run(input);
string inputLabel = dollar.Labels[0];
string modelLabel = model.Label;
bool isCorrect = inputLabel.Equals(modelLabel);
foreach (Sketch template in templates)
{
if (modelLabel.Equals(template.Label))
{
CorrectSymbol = SketchTools.Clone(template);
break;
}
}
return(isCorrect);
}
private bool StrokeBoundsTest(Sketch model, Sketch input)
{
// end test if there is a stroke count mismatch;
// can't compare stroke bounds if there are no corresponding strokes
if (!CheckStrokeCount(model, input))
{
return(false);
}
// get the cloned input and model sketches;
// necessary for not affecting use of input and model sketches in other methods
input = SketchTools.Clone(input);
model = SketchTools.Clone(model);
// get the matching input and model strokes
myStrokeMatches = GetStrokeMatches(input, model);;
// get the corresponding strokes and whether their bounds are proportionally correct
var triples = new List <Tuple <InkStroke, InkStroke, bool> >();
for (int i = 0; i < myStrokeMatches.Count; ++i)
{
// get the current corresponding strokes
var pair = myStrokeMatches[i];
InkStroke inputStroke = pair.Item1;
InkStroke modelStroke = pair.Item2;
// get the corresponding strokes' bounding boxes
Rect inputRect = inputStroke.BoundingRect;
Rect modelRect = modelStroke.BoundingRect;
// get the angles of the bounding boxes' diagonals
double inputAngle = Math.Atan(inputRect.Height / inputRect.Width) * 180 / Math.PI;
double modelAngle = Math.Atan(modelRect.Height / modelRect.Width) * 180 / Math.PI;
// case: the input or the model sketch has a vertical-line bounding box
bool isCorrect = false;
if (inputAngle < 10.0 || modelAngle < 10.0)
{
if (inputAngle >= 10.0 || modelAngle >= 10.0)
{
isCorrect = false;
}
else
{
double proportionRatio = 1.0 - Math.Abs(1.0 - (inputRect.Width / modelRect.Width));
isCorrect = proportionRatio > 0.85;
}
}
// case: the input or the model sketch as a horizontal-line bounding box
else if (inputAngle > 80.0 || modelAngle > 80.0)
{
if (inputAngle <= 80.0 || modelAngle <= 80.0)
{
isCorrect = false;
}
else
{
double proportionRatio = 1.0 - Math.Abs(1.0 - (inputRect.Height / modelRect.Height));
isCorrect = proportionRatio > 0.85;
}
}
// case: the input and model sketch both have regular bounding boxes
else
{
double inputProportion = inputRect.Width / inputRect.Height;
double modelProportion = modelRect.Width / modelRect.Height;
double proportionRatio = inputProportion / modelProportion;
isCorrect = proportionRatio < 1.3;
}
// collect the corresponding strokes' and the bounds correctness
var triple = new Tuple <InkStroke, InkStroke, bool>(inputStroke, modelStroke, isCorrect);
triples.Add(triple);
}
// check the bounds correctness for each corresponding strokes
foreach (var triple in triples)
{
bool isCorrect = triple.Item3;
if (!isCorrect)
{
return(false);
}
}
return(true);
}
private async void MyStrokeSpeedTestPlayButton_Click(object sender, RoutedEventArgs e)
{
// do not show feedback if canvas has no strokes or incorrect stroke count
if (MyInkStrokes.GetStrokes().Count <= 0)
{
return;
}
if (!myTechniqueRecognizer.StrokeCountResult)
{
return;
}
//
EnableTechniqueButtons(false);
//
bool hasInput = false;
Sketch model = SketchTools.Clone(myTemplates[MyCurrentIndex]);
Sketch input = null;
if (MyInkStrokes.GetStrokes().Count > 0)
{
input = new Sketch("", new List <InkStroke>(MyInkStrokes.GetStrokes()), myTimeCollection, 0, 0, BorderLength, BorderLength);
hasInput = true;
}
// get the converted model times
long modelOffset = model.Times[0][0];
long modelShift = 0;
List <List <long> > modelTimesCollection = new List <List <long> >();
long modelFactor = 2;
for (int i = 0; i < model.Times.Count; ++i)
{
//
List <long> newModelTimes = new List <long>();
foreach (long modelTime in model.Times[i])
{
long newModelTime = (modelTime - modelOffset - modelShift) / modelFactor;
newModelTimes.Add(newModelTime);
}
//
if (i < model.Times.Count - 1)
{
long nextStart = model.Times[i + 1][0];
long prevLast = model.Times[i][model.Times[i].Count - 1];
modelShift += nextStart - prevLast;
}
modelTimesCollection.Add(newModelTimes);
}
//
model = new Sketch(model.Label, model.Strokes, modelTimesCollection, model.FrameMinX, model.FrameMinY, model.FrameMaxX, model.FrameMaxY);
SolidColorBrush modelBrush = new SolidColorBrush(Colors.Black)
{
Opacity = 0.8
};
List <Storyboard> modelStoryboards = InteractionTools.Playback(MyCanvas, model.Strokes, model.Times, LARGE_DOT_SIZE, modelBrush);
foreach (Storyboard storyboard in modelStoryboards)
{
storyboard.Begin();
}
//
if (!hasInput)
{
return;
}
input = SketchTools.Clone(input);
SolidColorBrush inputBrush = new SolidColorBrush(Colors.Red)
{
Opacity = 1.0
};
List <Storyboard> inputStoryboards = InteractionTools.Playback(MyCanvas, input.Strokes, input.Times, SMALL_DOT_SIZE, inputBrush);
foreach (Storyboard storyboard in inputStoryboards)
{
storyboard.Begin();
}
//
int inputDelay = (int)((input.Times[input.Times.Count - 1])[input.Times[input.Times.Count - 1].Count - 1] - (input.Times[0])[0]);
int modelDelay = (int)((model.Times[model.Times.Count - 1])[model.Times[model.Times.Count - 1].Count - 1] - (model.Times[0])[0]);
int delay = inputDelay > modelDelay ? inputDelay : modelDelay;
await InteractionTools.Delay(delay);
//
EnableTechniqueButtons(true);
}
private async void MyStrokeDirectionPlayButton_Click(object sender, RoutedEventArgs e)
{
// do not show feedback if canvas has no strokes or incorrect stroke count
if (MyInkStrokes.GetStrokes().Count <= 0)
{
return;
}
if (!myTechniqueRecognizer.StrokeCountResult)
{
return;
}
//
EnableTechniqueButtons(false);
// get the input sketch and stroke directions
Sketch input = Sketch.CreateStroke("", new List <InkStroke>(MyInkStrokes.GetStrokes()), myTimeCollection, 0, 0, BorderLength, BorderLength);
input = SketchTools.Clone(input);
input = SketchTransformation.Resample(input, 128);
List <bool> strokeDirections = new List <bool>(myTechniqueRecognizer.StrokeDirections);
// create solution
InkStrokeBuilder builder = new InkStrokeBuilder();
List <InkStroke> solutionStrokes = new List <InkStroke>();
for (int i = 0; i < input.Strokes.Count; ++i)
{
// get the current input stroke and times
InkStroke inputStroke = input.Strokes[i];
List <long> inputTimes = input.Times[i];
// get current input points and stroke direction status
List <InkPoint> inputPoints = new List <InkPoint>(inputStroke.GetInkPoints());
bool strokeDirection = strokeDirections[i];
// create the solution stroke
List <Point> solutionPoints = new List <Point>();
foreach (InkPoint inputPoint in inputPoints)
{
solutionPoints.Add(new Point(inputPoint.Position.X, inputPoint.Position.Y));
}
if (!strokeDirection)
{
solutionPoints.Reverse();
}
InkStroke solutionStroke = builder.CreateStroke(solutionPoints);
// add the solution stroke to the list
solutionStrokes.Add(solutionStroke);
}
// display solution
SolidColorBrush newInputBrush = new SolidColorBrush(Colors.Black)
{
Opacity = 1.0
};
List <Storyboard> newInputStoryboards = InteractionTools.Trace(MyCanvas, solutionStrokes, input.Times, LARGE_DOT_SIZE, newInputBrush, POINT_DURATION);
foreach (Storyboard storyboard in newInputStoryboards)
{
storyboard.Begin();
}
// display original
SolidColorBrush inputBrush = new SolidColorBrush(Colors.Red)
{
Opacity = 1.0
};
List <Storyboard> inputStoryboards = InteractionTools.Trace(MyCanvas, input.Strokes, input.Times, LARGE_DOT_SIZE, inputBrush, POINT_DURATION);
foreach (Storyboard storyboard in inputStoryboards)
{
storyboard.Begin();
}
// re-enable return button
int delay = 0;
foreach (InkStroke inputStroke in input.Strokes)
{
delay += POINT_DURATION * inputStroke.GetInkPoints().Count;
}
await InteractionTools.Delay(delay);
//
EnableTechniqueButtons(true);
}
private async void MyPlayButton_Click(object sender, RoutedEventArgs e)
{
// get the input and model sketch, and set the duration
List <InkStroke> strokes = new List <InkStroke>();
foreach (InkStroke stroke in MyInkStrokes.GetStrokes())
{
strokes.Add(stroke);
}
Sketch input = new Sketch("", strokes, myTimeCollection, 0, 0, MyBorderLength, MyBorderLength);
Sketch model = myTemplates[MyImageIndex];
int duration = 30000;
// animate the expert's model strokes
if (MyImageButton.IsChecked.Value)
{
Sketch sketch = SketchTools.Clone(model);
double opacity = strokes.Count > 0 ? 0.8 : 1.0;
Color color = strokes.Count > 0 ? Colors.Black : Colors.Green;
SolidColorBrush brush = new SolidColorBrush(color)
{
Opacity = opacity
};
List <Storyboard> modelStoryboards = InteractionTools.Trace(MyCanvas, sketch.Strokes, sketch.Times, brush, duration);
foreach (Storyboard storyboard in modelStoryboards)
{
storyboard.Begin();
}
}
// animate the user's input strokes
if (strokes.Count > 0)
{
Sketch sketch = SketchTools.Clone(input);
sketch = SketchTransformation.Resample(sketch, 128);
double opacity = 1.0;
Color color = Colors.Red;
SolidColorBrush brush = new SolidColorBrush(color)
{
Opacity = opacity
};
List <Storyboard> inputStoryboards = InteractionTools.Trace(MyCanvas, sketch.Strokes, sketch.Times, brush, duration, model);
foreach (Storyboard storyboard in inputStoryboards)
{
storyboard.Begin();
}
}
//
if (strokes.Count == 0 && !MyImageButton.IsChecked.Value)
{
return;
}
//
int numModelPoints = 0;
foreach (InkStroke stroke in model.Strokes)
{
numModelPoints += stroke.GetInkPoints().Count;
}
int delay = (numModelPoints * duration) / 10000;
MyPlayButton.IsEnabled = false;
MyInkCanvas.InkPresenter.IsInputEnabled = false;
await Task.Delay(delay);
MyPlayButton.IsEnabled = true;
MyInkCanvas.InkPresenter.IsInputEnabled = true;
}
