/// <summary>
/// Network learns new curve
/// (adds curve to collection and runs training )
/// </summary>
/// <param name="curve">Curve</param>
public void LearnNewCurve(ClassicCurve curve)
{
int position = FindFreePosition();
if (position == -1)
{
// maximum amount of gestures that can be learnt is reached
return;
}
StopLearning(); //just for sure...
//if (threadTrainNetwork != null && threadTrainNetwork.IsAlive)
// threadTrainNetwork.Abort();
//if (m_newCurveAdded) m_curves.RemoveAt(m_curves.Count - 1);
MyCurve curveToTrain = new MyCurve(curve.ID, curve.Points);
curve.NnIndex = position;
curveToTrain.CreateTrainingSet();
// Might happen that curve sets is not compatible with curves
// therefore it is required to reinicialize it
if (m_curveSets.ContainsKey(curve.ID))
{
CreateTrainingSet();
}
m_curveSets.Add(curve.ID, curveToTrain);
m_curves.Add(curve.ID, curve);
InitializeNetwork();
OnNetworkStartTraining();
threadTrainNetwork = new Thread(new ThreadStart(TrainNeuralNetwork));
threadTrainNetwork.Start();
}
/// <summary>
/// Analyze the drawn curve and find associated learnt one
/// </summary>
private void StartAnalyze()
{
//form_top.MakeNonTopMost();
//if (cMS_MatchedGestures.Visible)
// form_top.CloseAndClearContextMenu(cMS_MatchedGestures);
Debug.WriteLine("Thread StartAnalyze started");
int approxPoints = Config.User.NnInputSize / 2 + 1;
double[] nnInput = null;
if (m_curve == null)
{
return;
}
else
{
//Get scaled input cannot last to long otherwise might be modified with new curve
try { nnInput = m_curve.GetScaledInput(); }
catch (Exception ex) { nnInput = null; }
}
m_curve = null;
if (nnInput == null)
{
return;
}
double divergence;
string nn_name = m_network.RecognizeCurve(nnInput, out divergence);
m_threadID = Thread.CurrentThread.ManagedThreadId;
GesturesCollection recognizedGestures = GetRecognizedGestures(nn_name);
ExecuteRecognizedGestures(recognizedGestures);
}
private void pB_display_MouseMove(object sender, MouseEventArgs e)
{
if (m_mouseDown)
{
Point point = new Point(e.X, e.Y);
m_pathPoints.Add(point);
m_curveLength += MyCurve.Distance(m_pathPoints[m_pathPoints.Count - 2], m_pathPoints[m_pathPoints.Count - 1]);
Color penColor = m_pen.Color;
if (m_curveLength < CURVE_MIN_LENGTH || m_curveLength >= 1500)
{
m_pen.Color = Color.Red;
}
else if (m_curveLength >= 100 && m_curveLength < 300 ||
m_curveLength >= 1300 && m_curveLength < 1500)
{
m_pen.Color = Color.Orange;
}
else
{
m_pen.Color = Color.Green;
}
if (penColor != m_pen.Color)
{
DrawLine(m_pathPoints, m_pen, true);
}
else
{
m_gp.DrawLine(m_pen, m_pathPoints[m_pathPoints.Count - 1], m_pathPoints[m_pathPoints.Count - 2]);
}
pB_display.Invalidate();
}
}
private void CreateTrainingSet()
{
m_curveSets = new Dictionary <string, MyCurve>();
foreach (ClassicCurve curve in m_curves.Values)// ClassicCurve curve in m_curves)
{
MyCurve c = new MyCurve(curve.ID, curve.Points);
c.CreateTrainingSet();
m_curveSets.Add(curve.ID, c);
}
}
private void btn_generate_Click(object sender, EventArgs e)
{
m_gp = Graphics.FromImage(pB_display.Image);
m_gp.FillRectangle(Brushes.White, 0, 0, pB_display.Width, pB_display.Height);
m_pen.Color = Color.Red;
int count = Int32.Parse(tB_approxPoints.Text);
List <PointF> exactPath = MyCurve.CreateExactPath(m_pathPoints, Int32.Parse(tB_exactPathMin.Text), Int32.Parse(tB_exactPathMax.Text));
//PointF one = new PointF(exactPath[0].X, exactPath[0].Y);
//PointF two = new PointF(exactPath[1].X, exactPath[1].Y);
//PointF last = new PointF(exactPath[exactPath.Count - 1].X, exactPath[exactPath.Count - 1].Y);
//PointF beforeLast = new PointF(exactPath[exactPath.Count - 2].X, exactPath[exactPath.Count - 2].Y);
//exactPath.Insert(0, new PointF(one.X - (two.X - one.X), one.Y - (two.Y - one.Y)));
//exactPath.Add(new PointF(last.X + (last.X - beforeLast.X), last.Y + (last.Y - beforeLast.Y)));
List <PointF> generated = MyCurve.GenerateFalseCurve(exactPath);
//List<PointF> generated = MyCurve.GenerateCurve(exactPath);
//List<PointF> bezierCurve = MyCurve.CreateBezierCurve(m_pathPoints);
List <PointF> approxPoints = MyCurve.ApproximateWithPoints(generated, count);
if (cB_originalCurve.Checked)
{
DrawLine(m_pathPoints, m_pen, true);
}
if (cB_curvePoints.Checked)
{
DrawLine(m_pathPoints, new Pen(Brushes.Yellow, 4), false);
}
//if (cB_bezierCurve.Checked)
// DrawLine(bezierCurve, new Pen(Brushes.Blue, 2), true);
if (cB_generated.Checked)
{
DrawLine(generated, new Pen(Brushes.Blue, 4), false);
}
if (cB_exactPath.Checked)
{
DrawLine(exactPath, new Pen(Brushes.Black, 4), false);
}
if (cB_approxPoints.Checked)
{
DrawLine(approxPoints, new Pen(Brushes.Green, 2), true);
}
pB_display.Invalidate();
m_gp.Dispose();
}
/// <summary>
/// Method called when mouse is down and moved within the timeout => curve gesture
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
void MyMouse_MouseDownGesture(object sender, MouseEventArgs e)
{
// currently this method is called only for right mouse button
Debug.WriteLine(string.Format("MyEngine: -{0}- Mouse button -GESTURE DOWN- at X: {1} , Y: {2}", e.Button.ToString().ToUpper(), e.X, e.Y));
if (e.Button == m_toggleBtn && Config.User.UsingClassicCurve)
{
form_top.CloseContextMenuAsync();
m_cursorPosition = e.Location;
m_hwndForeWnd = Win32.GetForegroundWindow();
//m_hwndWndToUse = GetWindowToUse(e.Location);
m_toggleBtnPushed = true;
form_top.Mouse_ToggleButtonDown(sender, e);
m_curve = new MyCurve();
}
}
void MyMouse_MouseUp(object sender, MouseEventArgs e)
{
Debug.WriteLine(string.Format("MyEngine: -{0}- Mouse button -UP- at X: {1} , Y: {2}", e.Button.ToString().ToUpper(), e.X, e.Y));
if (e.Button == m_toggleBtn && m_toggleBtnPushed)
{
m_toggleBtnPushed = false;
form_top.Mouse_ToggleButtonUp(sender, e);
if (form_top.Points.Count <= 1)
{
return;
}
m_curve = new MyCurve(form_top.Points, false);
Thread threadAnalyzeGesture = new Thread(new ThreadStart(StartAnalyze));
threadAnalyzeGesture.Name = "Analyze Gesture";
threadAnalyzeGesture.Start();
//StartAnalyze();
}
else
{
IntPtr hwnd = Win32.WindowFromPoint(Cursor.Position);
// click on context menu only if it is Left or Right button
if (m_leftMouseDownOnContextMenu && e.Button == MouseButtons.Left)
{
if (form_top.IsContextMenu(hwnd))
{
form_top.PerformClickOnContextMenu();
}
m_leftMouseDownOnContextMenu = false;
((ExtraMouseHook)sender).CancelEvent = true;
}
else if (m_rightMouseDownOnContextMenu && e.Button == MouseButtons.Right)
{
if (form_top.IsContextMenu(hwnd))
{
form_top.PerformClickOnContextMenu();
}
m_rightMouseDownOnContextMenu = false;
((ExtraMouseHook)sender).CancelEvent = true;
}
}
}
void MyMouse_MouseStoped(object sender, EventArgs e)
{
if (form_top.Points.Count <= 1)
{
return;
}
m_curve = new MyCurve(form_top.Points, false);
int approxPoints = Config.User.NnInputSize / 2 + 1;
double[] nnInput = m_curve.GetScaledInput();
if (nnInput == null)
{
form_top.ShowToolTip(Unknown_gesture);
return;
}
double divergence;
string nn_name = m_network.RecognizeCurve(nnInput, out divergence);
ShowToolTip(nn_name);
}
private void RedrawAll()
{
m_gp = Graphics.FromImage(pB_display.Image);
m_gp.FillRectangle(Brushes.White, 0, 0, pB_display.Width, pB_display.Height);
m_pen.Color = Color.Red;
int count = Int32.Parse(tB_approxPoints.Text);
List <PointF> bezierCurve = MyCurve.CreateBezierCurve(m_pathPoints);
List <PointF> approxPoints = MyCurve.ApproximateWithPoints(m_pathPoints, count);
List <PointF> exactPath = MyCurve.CreateExactPath(m_pathPoints, Int32.Parse(tB_exactPathMin.Text), Int32.Parse(tB_exactPathMax.Text));
if (cB_originalCurve.Checked)
{
DrawLine(m_pathPoints, m_pen, true);
DrawLine(m_pathPoints, new Pen(Brushes.Black, 8), false);
}
if (cB_curvePoints.Checked)
{
DrawLine(m_pathPoints, new Pen(Brushes.Yellow, 4), false);
}
if (cB_bezierCurve.Checked)
{
DrawLine(bezierCurve, new Pen(Brushes.Blue, 2), true);
}
if (cB_approxPoints.Checked)
{
DrawLine(approxPoints, new Pen(Brushes.Green, 6), false);
}
if (cB_exactPath.Checked)
{
DrawLine(exactPath, new Pen(Brushes.Black, 4), false);
}
pB_display.Invalidate();
m_gp.Dispose();
}
/// <summary>
/// Method called when mouse is down and moved within the timeout => curve gesture
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
void MyMouse_MouseDownGesture(object sender, MouseEventArgs e)
{
// currently this method is called only for right mouse button
Debug.WriteLine(string.Format("MyEngine: -{0}- Mouse button -GESTURE DOWN- at X: {1} , Y: {2}", e.Button.ToString().ToUpper(), e.X, e.Y));
if (e.Button == m_toggleBtn && Config.User.UsingClassicCurve)
{
form_top.CloseContextMenuAsync();
m_cursorPosition = e.Location;
m_hwndForeWnd = Win32.GetForegroundWindow();
//m_hwndWndToUse = GetWindowToUse(e.Location);
m_toggleBtnPushed = true;
form_top.Mouse_ToggleButtonDown(sender, e);
m_curve = new MyCurve();
}
}
/// <summary>
/// Returns the name of the recognized curve
/// </summary>
/// <param name="angles">Scaled curve to angles</param>
/// <param name="divergence">Divergence to original curve</param>
/// <returns></returns>
public string RecognizeCurve(double[] angles, out double divergence)
{
divergence = -1;
if (threadTrainNetwork != null && threadTrainNetwork.IsAlive)
{
Debug.WriteLine("MyNN cannot recognize in learning phase");
return string.Empty; //cannot recognize during learning phase
}
double[] output = m_network.Compute(angles);
//learnt error is smaller than 0.00001;
int pos = -2;
double total_error = 0;
Debug.Write("MyNN Output: ");
for (int i = 0; i < output.Length; i++)
{
Debug.Write(string.Format("{0} ", Math.Floor(output[i] * 1000) / 1000.0));
if (output[i] > MIN_ACCURACY)
{
if (pos == -2) pos = i;
else return string.Empty;
}
else
total_error += output[i];
//if (output[i] <= MIN_ACCURACY && output[i] >= MAX_ACCURACY)
// return string.Empty;
}
Debug.WriteLine(" ");
if (pos == -2) return string.Empty;
double result = output[pos] - total_error;
Debug.WriteLine(string.Format("Total Accuracy is: {0}", Math.Floor(result * 1000) / 1000.0));
if (result < MIN_TOTAL_ACCURACY) return string.Empty;
MyCurve recognizedCurve = null;
foreach (ClassicCurve curve in m_curves.Values)
if (curve.NnIndex == pos)
{
recognizedCurve = new MyCurve(curve.ID, curve.Points);
break;
}
if (recognizedCurve != null)
{
//During recognition only reduced curve is loaded
divergence = ComputeDivergence(angles, new double[][] { recognizedCurve.GetScaledInput() });
if (divergence < MAX_DIVERGENCE)
return recognizedCurve.ID;
else
{
divergence = -1;
return string.Empty;
}
}
else
return string.Empty;
}
/// <summary>
/// Network learns new curve
/// (adds curve to collection and runs training )
/// </summary>
/// <param name="curve">Curve</param>
public void LearnNewCurve(ClassicCurve curve)
{
int position = FindFreePosition();
if (position == -1)
{
// maximum amount of gestures that can be learnt is reached
return;
}
StopLearning(); //just for sure...
//if (threadTrainNetwork != null && threadTrainNetwork.IsAlive)
// threadTrainNetwork.Abort();
//if (m_newCurveAdded) m_curves.RemoveAt(m_curves.Count - 1);
MyCurve curveToTrain = new MyCurve(curve.ID, curve.Points);
curve.NnIndex = position;
curveToTrain.CreateTrainingSet();
// Might happen that curve sets is not compatible with curves
// therefore it is required to reinicialize it
if (m_curveSets.ContainsKey(curve.ID))
CreateTrainingSet();
m_curveSets.Add(curve.ID, curveToTrain);
m_curves.Add(curve.ID, curve);
InitializeNetwork();
OnNetworkStartTraining();
threadTrainNetwork = new Thread(new ThreadStart(TrainNeuralNetwork));
threadTrainNetwork.Start();
}
private void CreateTrainingSet()
{
m_curveSets = new Dictionary<string, MyCurve>();
foreach (ClassicCurve curve in m_curves.Values)// ClassicCurve curve in m_curves)
{
MyCurve c = new MyCurve(curve.ID, curve.Points);
c.CreateTrainingSet();
m_curveSets.Add(curve.ID, c);
}
}
/// <summary>
/// Analyze the drawn curve and find associated learnt one
/// </summary>
private void StartAnalyze()
{
//form_top.MakeNonTopMost();
//if (cMS_MatchedGestures.Visible)
// form_top.CloseAndClearContextMenu(cMS_MatchedGestures);
Debug.WriteLine("Thread StartAnalyze started");
int approxPoints = Config.User.NnInputSize / 2 + 1;
double[] nnInput = null;
if (m_curve == null) return;
else
{
//Get scaled input cannot last to long otherwise might be modified with new curve
try { nnInput = m_curve.GetScaledInput(); }
catch (Exception ex) { nnInput = null; }
}
m_curve = null;
if (nnInput == null) return;
double divergence;
string nn_name = m_network.RecognizeCurve(nnInput, out divergence);
m_threadID = Thread.CurrentThread.ManagedThreadId;
GesturesCollection recognizedGestures = GetRecognizedGestures(nn_name);
ExecuteRecognizedGestures(recognizedGestures);
}
void MyMouse_MouseStoped(object sender, EventArgs e)
{
if (form_top.Points.Count <= 1) return;
m_curve = new MyCurve(form_top.Points, false);
int approxPoints = Config.User.NnInputSize / 2 + 1;
double[] nnInput = m_curve.GetScaledInput();
if (nnInput == null)
{
form_top.ShowToolTip(Unknown_gesture);
return;
}
double divergence;
string nn_name = m_network.RecognizeCurve(nnInput, out divergence);
ShowToolTip(nn_name);
}
void MyMouse_MouseUp(object sender, MouseEventArgs e)
{
Debug.WriteLine(string.Format("MyEngine: -{0}- Mouse button -UP- at X: {1} , Y: {2}", e.Button.ToString().ToUpper(), e.X, e.Y));
if (e.Button == m_toggleBtn && m_toggleBtnPushed)
{
m_toggleBtnPushed = false;
form_top.Mouse_ToggleButtonUp(sender, e);
if (form_top.Points.Count <= 1) return;
m_curve = new MyCurve(form_top.Points, false);
Thread threadAnalyzeGesture = new Thread(new ThreadStart(StartAnalyze));
threadAnalyzeGesture.Name = "Analyze Gesture";
threadAnalyzeGesture.Start();
//StartAnalyze();
}
else
{
IntPtr hwnd = Win32.WindowFromPoint(Cursor.Position);
// click on context menu only if it is Left or Right button
if (m_leftMouseDownOnContextMenu && e.Button == MouseButtons.Left)
{
if (form_top.IsContextMenu(hwnd))
form_top.PerformClickOnContextMenu();
m_leftMouseDownOnContextMenu = false;
((ExtraMouseHook)sender).CancelEvent = true;
}
else if (m_rightMouseDownOnContextMenu && e.Button == MouseButtons.Right)
{
if (form_top.IsContextMenu(hwnd))
form_top.PerformClickOnContextMenu();
m_rightMouseDownOnContextMenu = false;
((ExtraMouseHook)sender).CancelEvent = true;
}
}
}
public MyCurve(MyCurve curve)
{
m_name = curve.m_name;
m_points.AddRange(curve.m_points.ToArray());
m_trainingSet = curve.m_trainingSet;
}
/// <summary>
/// Creates input and output for nerual network
/// </summary>
/// <param name="input"></param>
/// <param name="output"></param>
private void CreateInputOutput(out double[][] input, out double[][] output)
{
int maxPos = -1;
string[] positions = new string[m_outputSize];
List <ClassicCurve> curves = new List <ClassicCurve>(m_curves.Values);
for (int i = 0; i < positions.Length; i++)
{
positions[i] = string.Empty;
}
for (int i = 0; i < curves.Count; i++)
{
int nnIndex = curves[i].NnIndex;
positions[nnIndex] = curves[i].ID;
maxPos = maxPos > nnIndex ? maxPos : nnIndex;
}
maxPos++; //we need to increment the position so we can reach it in a cycle (as on this position some gesture actualy is!)
List <double[]> inputList = new List <double[]>();
List <double[]> outputList = new List <double[]>();
for (int i = 0; i < maxPos; i++)
{
if (positions[i] != string.Empty)
{
inputList.AddRange(m_curveSets[positions[i]].TrainingSet);
}
else
{
inputList.AddRange(MyCurve.EmptyTrainingSet());
}
}
//input = inputList.ToArray();
int k = -1;
int trainingSize = m_setSize * maxPos;//m_curves.Count;
//output = new double[trainingSize][];
for (int i = 0; i < trainingSize; i++)
{
if (i % m_setSize == 0)
{
k++;
}
double[] oneOutput = new double[m_outputSize];
//output[i] = new double[m_outputSize];
if (positions[k] != string.Empty)
{
//output[i][m_curves[positions[k]].NnIndex] = 1;
oneOutput[m_curves[positions[k]].NnIndex] = 1;
}
outputList.Add(oneOutput);
}
//add false patterns
//double[] emptyOutput = new double[m_outputSize];
//foreach (MyCurve c in m_curveSets.Values)
//{
// double[][] falsePatterns = c.GenerateFalsePatterns();
// for (int i = 0; i < falsePatterns.Length; i++)
// {
// inputList.Add(falsePatterns[i]);
// outputList.Add(emptyOutput);
// }
//}
//shuffle it
List <double[]> tempInput = inputList; //new List<double[]>(input);
List <double[]> tempOutput = outputList; //new List<double[]>(output);
List <double[]> randInput = new List <double[]>();
List <double[]> randOutput = new List <double[]>();
while (tempInput.Count > 0)
{
int i = StaticRandom.RandomInteger(0, tempInput.Count);
randInput.Add(tempInput[i]);
randOutput.Add(tempOutput[i]);
tempInput.RemoveAt(i);
tempOutput.RemoveAt(i);
}
input = randInput.ToArray();
output = randOutput.ToArray();
}
/// <summary>
/// Returns the name of the recognized curve
/// </summary>
/// <param name="angles">Scaled curve to angles</param>
/// <param name="divergence">Divergence to original curve</param>
/// <returns></returns>
public string RecognizeCurve(double[] angles, out double divergence)
{
divergence = -1;
if (threadTrainNetwork != null && threadTrainNetwork.IsAlive)
{
Debug.WriteLine("MyNN cannot recognize in learning phase");
return(string.Empty); //cannot recognize during learning phase
}
double[] output = m_network.Compute(angles);
//learnt error is smaller than 0.00001;
int pos = -2;
double total_error = 0;
Debug.Write("MyNN Output: ");
for (int i = 0; i < output.Length; i++)
{
Debug.Write(string.Format("{0} ", Math.Floor(output[i] * 1000) / 1000.0));
if (output[i] > MIN_ACCURACY)
{
if (pos == -2)
{
pos = i;
}
else
{
return(string.Empty);
}
}
else
{
total_error += output[i];
}
//if (output[i] <= MIN_ACCURACY && output[i] >= MAX_ACCURACY)
// return string.Empty;
}
Debug.WriteLine(" ");
if (pos == -2)
{
return(string.Empty);
}
double result = output[pos] - total_error;
Debug.WriteLine(string.Format("Total Accuracy is: {0}", Math.Floor(result * 1000) / 1000.0));
if (result < MIN_TOTAL_ACCURACY)
{
return(string.Empty);
}
MyCurve recognizedCurve = null;
foreach (ClassicCurve curve in m_curves.Values)
{
if (curve.NnIndex == pos)
{
recognizedCurve = new MyCurve(curve.ID, curve.Points);
break;
}
}
if (recognizedCurve != null)
{
//During recognition only reduced curve is loaded
divergence = ComputeDivergence(angles, new double[][] { recognizedCurve.GetScaledInput() });
if (divergence < MAX_DIVERGENCE)
{
return(recognizedCurve.ID);
}
else
{
divergence = -1;
return(string.Empty);
}
}
else
{
return(string.Empty);
}
}
public MyCurve(MyCurve curve)
{
m_name = curve.m_name;
m_points.AddRange(curve.m_points.ToArray());
m_trainingSet = curve.m_trainingSet;
}
