private void PerformMouseCommand(SpeechRecognizedEventArgs e)
{
Win32.POINT p = new Win32.POINT();
p.x = 100;
p.y = 100;
var horizontalCoordinate = e.Result.Words[1].Text;
if (horizontalCoordinate == "Zero")
{
p.x = 5;
}
else if (horizontalCoordinate == "Alpha")
{
p.x = 50;
}
else if (horizontalCoordinate == "Bravo")
{
p.x = 100;
}
else if (horizontalCoordinate == "Charlie")
{
p.x = 150;
}
else if (horizontalCoordinate == "Delta")
{
p.x = 200;
}
else if (horizontalCoordinate == "Echo")
{
p.x = 250;
}
else if (horizontalCoordinate == "Foxtrot")
{
p.x = 300;
}
else if (horizontalCoordinate == "Golf")
{
p.x = 350;
}
else if (horizontalCoordinate == "Hotel")
{
p.x = 400;
}
else if (horizontalCoordinate == "India")
{
p.x = 450;
}
else if (horizontalCoordinate == "Juliet")
{
p.x = 500;
}
else if (horizontalCoordinate == "Kilo")
{
p.x = 550;
}
else if (horizontalCoordinate == "Lima")
{
p.x = 600;
}
else if (horizontalCoordinate == "Mike")
{
p.x = 650;
}
else if (horizontalCoordinate == "November")
{
p.x = 700;
}
else if (horizontalCoordinate == "Oscar")
{
p.x = 750;
}
else if (horizontalCoordinate == "Papa")
{
p.x = 800;
}
else if (horizontalCoordinate == "Qubec")
{
p.x = 850;
}
else if (horizontalCoordinate == "Romeo")
{
p.x = 900;
}
else if (horizontalCoordinate == "Sierra")
{
p.x = 950;
}
else if (horizontalCoordinate == "Tango")
{
p.x = 1000;
}
else if (horizontalCoordinate == "Uniform")
{
p.x = 1050;
}
else if (horizontalCoordinate == "Victor")
{
p.x = 1100;
}
else if (horizontalCoordinate == "Whiskey")
{
p.x = 1150;
}
else if (horizontalCoordinate == "X-ray")
{
p.x = 1200;
}
else if (horizontalCoordinate == "Yankee")
{
p.x = 1250;
}
else if (horizontalCoordinate == "Zulu")
{
p.x = 1300;
}
else if (horizontalCoordinate == "1")
{
p.x = 1350;
}
else if (horizontalCoordinate == "2")
{
p.x = 1400;
}
else if (horizontalCoordinate == "3")
{
p.x = 1450;
}
else if (horizontalCoordinate == "4")
{
p.x = 1500;
}
else if (horizontalCoordinate == "5")
{
p.x = 1550;
}
else if (horizontalCoordinate == "6")
{
p.x = 1600;
}
else if (horizontalCoordinate == "7")
{
p.x = 1650;
}
var verticalCoordinate = e.Result.Words[2].Text;
if (verticalCoordinate == "Zero")
{
p.y = 5;
}
else if (verticalCoordinate == "Alpha")
{
p.y = 50;
}
else if (verticalCoordinate == "Bravo")
{
p.y = 100;
}
else if (verticalCoordinate == "Charlie")
{
p.y = 150;
}
else if (verticalCoordinate == "Delta")
{
p.y = 200;
}
else if (verticalCoordinate == "Echo")
{
p.y = 250;
}
else if (verticalCoordinate == "Foxtrot")
{
p.y = 300;
}
else if (verticalCoordinate == "Golf")
{
p.y = 350;
}
else if (verticalCoordinate == "Hotel")
{
p.y = 400;
}
else if (verticalCoordinate == "India")
{
p.y = 450;
}
else if (verticalCoordinate == "Juliet")
{
p.y = 500;
}
else if (verticalCoordinate == "Kilo")
{
p.y = 550;
}
else if (verticalCoordinate == "Lima")
{
p.y = 600;
}
else if (verticalCoordinate == "Mike")
{
p.y = 650;
}
else if (verticalCoordinate == "November")
{
p.y = 700;
}
else if (verticalCoordinate == "Oscar")
{
p.y = 750;
}
else if (verticalCoordinate == "Papa")
{
p.y = 800;
}
else if (verticalCoordinate == "Qubec")
{
p.y = 850;
}
else if (verticalCoordinate == "Romeo")
{
p.y = 900;
}
else if (verticalCoordinate == "Sierra")
{
p.y = 950;
}
else if (verticalCoordinate == "Tango")
{
p.y = 1000;
}
else if (verticalCoordinate == "Uniform")
{
p.y = 1050;
}
else if (verticalCoordinate == "Victor")
{
p.y = 1100;
}
else if (verticalCoordinate == "Whiskey")
{
p.y = 1150;
}
else if (verticalCoordinate == "X-ray")
{
p.y = 1200;
}
else if (verticalCoordinate == "Yankee")
{
p.y = 1250;
}
else if (verticalCoordinate == "Zulu")
{
p.y = 1300;
}
else if (verticalCoordinate == "1")
{
p.y = 1350;
}
else if (verticalCoordinate == "2")
{
p.y = 1400;
}
else if (verticalCoordinate == "3")
{
p.y = 1450;
}
else if (verticalCoordinate == "4")
{
p.y = 1500;
}
else if (verticalCoordinate == "5")
{
p.y = 1550;
}
else if (verticalCoordinate == "6")
{
p.y = 1600;
}
else if (verticalCoordinate == "7")
{
p.y = 1650;
}
var screen = e.Result.Words[0].Text;
if (screen == "Right" || screen == "Touch")
{
p.x += 1680;
}
Win32.SetCursorPos(p.x, p.y);
SpeechUI.SendTextFeedback(e.Result, $" {e.Result.Text} H{p.x} V{p.y}", true);
if (screen == "Click" || screen == "Touch")
{
Win32.mouse_event(MOUSEEVENTF_LEFTDOWN | MOUSEEVENTF_LEFTUP, (uint)p.x, (uint)p.y, 0, 0);
}
}
protected async Task NewMemoSpeech()
{
string message;
try
{
this.SpeechUI.Settings.ShowConfirmation = false;
message = CensorProfanity((await SpeechUI.RecognizeWithUIAsync()).RecognitionResult.Text);
}
catch
{
Conversation.Add(new Message("Hello", MessageSide.User));
return;
}
// Got call back.
Conversation.Add(new Message(message, MessageSide.User));
message = message.Trim('.', '?', '!').ToLowerInvariant();
WebClient client = new WebClient();
if (this.IsTeach)
{
Conversation.Add(new Message(String.Format("Adding memo {0}.", message), MessageSide.Cloud));
BeginScrollDown();
await this.Synthesizer.SpeakTextAsync(String.Format("Adding memo {0}.", message));
// Call API
client.Headers[HttpRequestHeader.Accept] = "application/json";
client.DownloadStringAsync(
new Uri(String.Format(Api.TimeExtractionUri, HttpUtility.UrlEncode(message), DateTime.Now.ToString("s"))));
// Get time extraction response
client.DownloadStringCompleted += async(sender, e) =>
{
if (e.Error == null)
{
DateTime[] time = await JsonConvert.DeserializeObjectAsync <DateTime[]>(e.Result);
if (time.Length > 0)
{
SetupPopup(message, time[0], true);
}
else
{
SetupPopup(message, DateTime.Now + new TimeSpan(0, 30, 0), false);
}
}
else
{
SetupPopup(message, DateTime.Now + new TimeSpan(0, 30, 0), false);
}
};
// Get geo-position
Geoposition geoposition;
if ((bool)IsolatedStorageSettings.ApplicationSettings["LocationConsent"] == true)
{
Geolocator geolocator = new Geolocator();
geolocator.DesiredAccuracyInMeters = 50;
try
{
geoposition = await geolocator.GetGeopositionAsync(
maximumAge : TimeSpan.FromMinutes(5),
timeout : TimeSpan.FromSeconds(10)
);
this.CurrentLatitude = geoposition.Coordinate.Latitude;
this.CurrentLongitude = geoposition.Coordinate.Longitude;
}
catch
{
this.CurrentLatitude = null;
this.CurrentLongitude = null;
}
}
}
else
{
client.Headers[HttpRequestHeader.ContentType] = "application/json";
client.Headers[HttpRequestHeader.Accept] = "application/json";
client.UploadStringAsync(
new Uri(String.Format(Api.RetrieveUri, Phone.ID)),
JsonConvert.SerializeObject(new
{
content = message
}));
client.UploadStringCompleted += async(sender, args) =>
{
if (args.Error == null)
{
string[] results = JsonConvert.DeserializeObject <string[]>(args.Result);
if (results.Length > 0)
{
IAsyncAction resultAction = this.Synthesizer.SpeakTextAsync(results[0]);
this.Conversation.Add(new Message(results[0], MessageSide.Cloud));
this.BeginScrollDown();
await resultAction;
}
else
{
IAsyncAction resultAction = this.Synthesizer.SpeakTextAsync("Can't recall \"" + message + "\".");
this.Conversation.Add(new Message("Can't recall \"" + message + "\".", MessageSide.Cloud));
this.BeginScrollDown();
await resultAction;
}
}
else
{
await HandleInternetException();
}
};
}
}
public async void PerformCommand(SpeechRecognizedEventArgs e, AvailableCommandsForm form, SpeechRecognizer speechRecogniser, DictateSpeech dictateSpeech, Microsoft.CognitiveServices.Speech.SpeechRecognizer speechRecognizer)
{
UpdateCurrentProcess();
try
{
SpeechUI.SendTextFeedback(e.Result, $"Recognised: {e.Result.Text} {e.Result.Confidence:P1}", true);
}
catch (Exception exception)
{
AutoClosingMessageBox.Show(exception.Message, "Error Sending feedback to speech recognition", 3000);
}
if (e.Result.Grammar.Name == "New with Space" && e.Result.Confidence > 0.6)
{
inputSimulator.Keyboard.TextEntry(" new ");
inputSimulator.Keyboard.KeyDown(VirtualKeyCode.ESCAPE);
}
else if (e.Result.Grammar.Name == "Step Over" && e.Result.Confidence > 0.6)
{
inputSimulator.Keyboard.KeyDown(VirtualKeyCode.F10);
}
else if (e.Result.Grammar.Name == "Step Into" && e.Result.Confidence > 0.6)
{
inputSimulator.Keyboard.KeyDown(VirtualKeyCode.F11);
}
else if (e.Result.Grammar.Name == "Reset Code" && e.Result.Confidence > 0.6)
{
inputSimulator.Keyboard.ModifiedKeyStroke(VirtualKeyCode.SHIFT, VirtualKeyCode.F5);
}
else if (e.Result.Grammar.Name == "Use Dragon" && e.Result.Confidence > 0.6)
{
ToggleSpeechRecognitionListeningMode(inputSimulator);
inputSimulator.Keyboard.KeyDown(VirtualKeyCode.ADD);
}
else if (e.Result.Grammar.Name == "Window Monitor Switch" && e.Result.Confidence > 0.6)
{
inputSimulator.Keyboard.ModifiedKeyStroke(windowAndShift, VirtualKeyCode.RIGHT);
}
else if (e.Result.Grammar.Name == "Select Line" && e.Result.Confidence > 0.6)
{
inputSimulator.Keyboard.KeyPress(VirtualKeyCode.HOME);
inputSimulator.Keyboard.ModifiedKeyStroke(VirtualKeyCode.SHIFT, VirtualKeyCode.END);
}
else if (e.Result.Grammar.Name == "Mouse Down" && e.Result.Confidence > 0.6)
{
inputSimulator.Mouse.LeftButtonDown();
}
else if (e.Result.Grammar.Name == "Shutdown Windows" && e.Result.Confidence > 0.5)
{
CommandToBeConfirmed = e.Result.Grammar.Name;
SetupConfirmationCommands(speechRecogniser, form);
}
else if (e.Result.Grammar.Name == "Restart Windows" && e.Result.Confidence > 0.5)
{
CommandToBeConfirmed = e.Result.Grammar.Name;
SetupConfirmationCommands(speechRecogniser, form);
}
else if (e.Result.Grammar.Name == "Confirmed")
{
if (CommandToBeConfirmed == "Shutdown Windows")
{
Process.Start("shutdown", "/s /t 10");
}
else if (CommandToBeConfirmed == "Restart Windows")
{
Process.Start("shutdown", "/r /t 10");
}
PerformQuitApplicationCommand(e);
}
else if (e.Result.Grammar.Name == "Short Dictation" && e.Result.Confidence > 0.4)
{
await PerformShortDictation(e, form, dictateSpeech, speechRecognizer);
}
else if (e.Result.Grammar.Name == "Serenade" && e.Result.Confidence > 0.4)
{
PerformSerenadeCommand(speechRecogniser);
}
else if (e.Result.Grammar.Name == "Denied")
{
var availableCommands = speechSetup.SetUpMainCommands(speechRecogniser, form.UseAzureSpeech);
form.RichTextBoxAvailableCommands = availableCommands;
}
else if (e.Result.Grammar.Name == "Studio" && e.Result.Confidence > 0.5)
{
RunVisualStudioCommand(speechRecogniser);
}
else if (e.Result.Grammar.Name == "Default Box" && e.Result.Confidence > 0.5)
{
Process.Start(@"C:\Users\MPhil\Source\Repos\SpeechRecognitionHelpers\DictationBoxMSP\bin\Release\DictationBoxMSP.exe");
}
else if (e.Result.Grammar.Name == "Dictation Box" && e.Result.Confidence > 0.5)
{
Process.Start(@"C:\Program Files (x86)\Speech Productivity\dictation box default\dictation box.exe");
}
else if (e.Result.Grammar.Name == "Get and Set" && e.Result.Confidence > 0.5)
{
inputSimulator.Keyboard.TextEntry(" { get; set; }");
}
else if (e.Result.Grammar.Name.Contains("Phonetic Alphabet")) // Could be lower, mixed or upper
{
ProcessKeyboardCommand(e);
}
else if (e.Result.Grammar.Name == "Show Recent" && e.Result.Confidence > 0.5)
{
inputSimulator.Keyboard.ModifiedKeyStroke(VirtualKeyCode.LMENU, VirtualKeyCode.VK_F);
inputSimulator.Keyboard.KeyPress(VirtualKeyCode.VK_J);
}
else if (e.Result.Grammar.Name == "Fresh Line" && e.Result.Confidence > 0.5)
{
inputSimulator.Keyboard.KeyPress(VirtualKeyCode.END);
inputSimulator.Keyboard.KeyPress(VirtualKeyCode.RETURN);
}
else if (e.Result.Grammar.Name == "Semi Colon" && e.Result.Confidence > 0.3)
{
inputSimulator.Keyboard.TextEntry(";");
}
// where the grammar name is the same Click cancelas the method Without the perform and command, with the spaces remove use reflection to call it
else if (e.Result.Confidence > 0.6)
{
string methodName = $"Perform{e.Result.Grammar.Name.Replace(" ", "")}Command";
Type thisType = this.GetType();
//MethodInfo theMethod = thisType.GetMethod(methodName,BindingFlags.NonPublic | BindingFlags.Instance);
try
{
thisType.InvokeMember(methodName,
BindingFlags.DeclaredOnly |
BindingFlags.Public | BindingFlags.NonPublic |
BindingFlags.Instance | BindingFlags.InvokeMethod
, null, this, new Object[] { e });
}
catch (Exception exception)
{
//AutoClosingMessageBox.Show(exception.Message, $"Error Running a method {exception.Source}", 3000);
System.Windows.Forms.MessageBox.Show(exception.Message, "Error running a method", MessageBoxButtons.OK);
}
}
}