protected void InitSpeechEngine(KinectSensor Sensor) { source = Sensor.AudioSource; source.EchoCancellationMode = EchoCancellationMode.CancellationAndSuppression; source.NoiseSuppression = true; source.BeamAngleMode = BeamAngleMode.Adaptive; // set the beam to adapt to the surrounding source.AutomaticGainControlEnabled = false; var echo = ConfigManager.GetInstance().Find("kinect_v1.speech.echo", -1); if (echo >= 0) { source.EchoCancellationSpeakerIndex = echo; } Log("AutomaticGainControlEnabled : " + source.AutomaticGainControlEnabled); Log("BeamAngle : " + source.BeamAngle); Log("EchoCancellationMode : " + source.EchoCancellationMode); Log("EchoCancellationSpeakerIndex : " + source.EchoCancellationSpeakerIndex); Log("NoiseSuppression : " + source.NoiseSuppression); Log("SoundSourceAngle : " + source.SoundSourceAngle); Log("SoundSourceAngleConfidence : " + source.SoundSourceAngleConfidence); var stream = source.Start(); double confidence = ConfigManager.GetInstance().Find("kinect_v1.speech.confidence", 0.6); AddOnManager.GetInstance().AddAudioSource(Name, stream, "Kinect", null, confidence); }
// ------------------------------------------- // UPLOAD // ------------------------------------------- public void SendFILE(string device, string url, string token, string path) { if (url == null) { return; } if (path == null) { SendGET(device, url, token, null); return; } url = CleanURL(url); Log("Build UploadRequest: " + url); WebClient client = new WebClient(); client.Headers.Add("user-agent", "S.A.R.A.H. (Self Actuated Residential Automated Habitat)"); try { byte[] responseArray = client.UploadFile(url, path); String response = System.Text.Encoding.ASCII.GetString(responseArray); AddOnManager.GetInstance().HandleBODY(device, response, token); } catch (Exception ex) { Error("Exception: " + ex.Message); } }
public void SpeechRecognized(SpeechEngine engine, RecognitionResult rr) { // 1. Handle the Listening global state if (!IsListening) { Log("REJECTED not listening"); return; } // Compute XPath Navigator XPathNavigator xnav = rr.ConstructSmlFromSemantics().CreateNavigator(); // 2. Handle confidence if (!Confidence(engine.Name, rr, xnav, engine.Confidence)) { return; } // 3. Set an engagement for valid audio AddOnManager.GetInstance().HandleProfile(engine.Name, "engaged", (Object)DateTime.Now); // 4. Forward to all addons var text = rr.Text; var grammar = rr.Grammar.Name; var options = new Dictionary <string, string>(); using (var stream = new MemoryStream()) { rr.Audio.WriteToWaveStream(stream); AddOnManager.GetInstance().BeforeSpeechRecognition(engine.Name, text, rr.Confidence, xnav, grammar, stream, options); AddOnManager.GetInstance().AfterSpeechRecognition(engine.Name, text, rr.Confidence, xnav, grammar, stream, options); } }
public void SpeechRejected(SpeechEngine engine, RecognitionResult rr) { // 1. Handle the Listening global state if (!IsListening) { Log("REJECTED not listening"); return; } // 2. Check DYN Grammar if (ContextManager.GetInstance().Dynamic() == null) { return; } XPathNavigator xnav = rr.ConstructSmlFromSemantics().CreateNavigator(); // 3. Forward to all addons var text = rr.Text; var options = new Dictionary <string, string>(); using (var stream = new MemoryStream()) { rr.Audio.WriteToWaveStream(stream); AddOnManager.GetInstance().BeforeSpeechRejected(engine.Name, text, rr.Confidence, xnav, stream, options); AddOnManager.GetInstance().AfterSpeechRejected(engine.Name, text, rr.Confidence, xnav, stream, options); } }
public void Speak(String tts, bool sync) { Log("Speaking: " + tts); try { PromptBuilder builder = new PromptBuilder(); builder.Culture = new CultureInfo(ConfigManager.GetInstance().Find("bot.language", "fr-FR")); builder.AppendText(tts); using (var ms = new MemoryStream()) { lock (synthesizer) { synthesizer.SetOutputToWaveStream(ms); synthesizer.Speak(builder); } ms.Position = 0; if (ms.Length <= 0) { return; } AddOnManager.GetInstance().AfterHandleVoice(tts, sync, ms); } } catch (Exception ex) { Error(ex); } }
protected override void DoTask() { var tmp = StandBy; Motion = CompareDepth(depth1, Depth.Pixels); Array.Copy(Depth.Pixels, depth1, depth1.Length); // Backup if (Motion > threshold) { StandByWatch.Restart(); StandBy = false; } else if (StandByWatch.Elapsed > timeout) { StandByWatch.Stop(); StandBy = true; } if (tmp != StandBy) { if (StandBy) { Array.Copy(depth1, NoMotion, NoMotion.Length); } AddOnManager.GetInstance().MotionDetected(Device, !StandBy); } }
protected void InitAudioWatcher() { if (audioWatcher != null) { return; } string directory = ConfigManager.GetInstance().Find("os.recognize", ""); if (!Directory.Exists(directory)) { return; } Log("Init Audio Watcher: " + directory); audioWatcher = new AdvancedFileSystemWatcher(); audioWatcher.Path = directory; audioWatcher.Filter = "*.wav"; audioWatcher.IncludeSubdirectories = true; audioWatcher.NotifyFilter = NotifyFilters.LastWrite; audioWatcher.Changed += new EventHandler <SoftMade.IO.FileSystemEventArgs>(audio_Changed); audioWatcher.EnableRaisingEvents = true; buffer = new StreamBuffer(); var confidence = ConfigManager.GetInstance().Find("os.confidence", 0.6); var format = ConfigManager.GetInstance().Find("os.format", "Kinect"); AddOnManager.GetInstance().AddAudioSource("FileSystem", buffer, format, null, confidence); }
public async Task LicenseFailureInRetail() { var testStorageDirectory = GetTestStorageDirectory(); try { using (var installation = new WorldOfWarcraftInstallation()) using (var manager = new AddOnManager(testStorageDirectory, installation)) { await manager.InitializationComplete; var blankAddOn = manager.AddOns.First(addOn => addOn.Key == blankAddOnKey); Assert.IsFalse(blankAddOn.IsDownloaded); Assert.IsFalse(blankAddOn.IsLicensed); await blankAddOn.DownloadAsync(); Assert.IsTrue(blankAddOn.IsDownloaded); Assert.IsTrue(blankAddOn.IsLicensed); Assert.IsFalse(blankAddOn.IsLicenseAgreed); Assert.IsFalse(blankAddOn.IsInstalled); await blankAddOn.InstallAsync(); } } finally { CleanTestStorageDirectory(testStorageDirectory); } }
public override void AfterSpeechRecognition(string device, string text, double confidence, XPathNavigator xnav, string grammar, Stream stream, IDictionary <string, string> options) { base.AfterSpeechRecognition(device, text, confidence, xnav, grammar, stream, options); // Reset context timeout ContextManager.GetInstance().ResetContextTimeout(); // Update Context var context = xnav.SelectSingleNode("/SML/action/@context"); if (context != null) { ContextManager.GetInstance().SetContext(context.Value); ContextManager.GetInstance().ApplyContextToEngines(); ContextManager.GetInstance().StartContextTimeout(); } // Update a Grammar with XML var asknext = xnav.SelectSingleNode("/SML/action/@asknext"); if (asknext != null) { var example = GrammarManager.GetInstance().FindExample(asknext.Value); AddOnManager.GetInstance().BeforeHandleVoice(example, true); } }
// ------------------------------------------ // Audio Management // ------------------------------------------ public override void BeforeSpeechRecognition(string device, string text, double confidence, XPathNavigator xnav, string grammar, Stream stream, IDictionary <string, string> options) { base.BeforeSpeechRecognition(device, text, confidence, xnav, grammar, stream, options); byte[] audioBytes = null; stream.Position = 0; using (MemoryStream audioStream = new MemoryStream()) { stream.CopyTo(audioStream); audioStream.Position = 0; audioBytes = audioStream.ToArray(); } float[] audioBuffer = new float[audioBytes.Length / 2]; for (int i = 0, j = 0; i < audioBytes.Length / 2; i += 2, j++) { // convert two bytes to one short short s = BitConverter.ToInt16(audioBytes, i); // convert to range from -1 to (just below) 1 audioBuffer[j] = s / 32768.0f; } // Reset tracker.Reset(); pitch.Clear(); // Process tracker.ProcessBuffer(audioBuffer); // Notify AddOnManager.GetInstance().HandleProfile(device, "pitch", pitch.Mean()); }
public override void AfterHTTPRequest(string qs, NameValueCollection parameters, IDictionary files) { base.AfterHTTPRequest(qs, parameters, files); var querystring = System.Web.HttpUtility.ParseQueryString(qs); // Update Context var context = querystring.GetValues("context"); if (context != null) { ContextManager.GetInstance().SetContext(context); ContextManager.GetInstance().ApplyContextToEngines(); ContextManager.GetInstance().StartContextTimeout(); } // Update a Grammar with XML var grammar = parameters.Get("grammar"); if (grammar != null) { GrammarManager.GetInstance().UpdateXML(grammar, parameters.Get("xml")); } // AskMe: start a Dynamic Grammar String[] sentences = querystring.GetValues("sentences"); String[] tags = querystring.GetValues("tags"); String callbackUrl = parameters.Get("callbackUrl"); if (sentences != null && tags != null) { ContextManager.GetInstance().LoadDynamicGrammar(sentences, tags, callbackUrl); ContextManager.GetInstance().SetContext("Dyn"); ContextManager.GetInstance().ApplyContextToEngines(); } // AskNext: TTS Acording to ruleId var asknext = parameters.Get("asknext"); if (asknext != null) { var example = GrammarManager.GetInstance().FindExample(asknext); if (example != null) { var regexp = new Regex("\\[([^\\]]+)\\]"); var matches = regexp.Matches(example); foreach (Match match in matches) { var key = match.Groups[1]; var value = parameters.Get(key.ToString()); if (value == null) { value = ""; } example = example.Replace("[" + key + "]", value); } AddOnManager.GetInstance().BeforeHandleVoice(example, true); } } }
public async Task AddOnManifestLoadsAsync() { using (var manager = new AddOnManager(null, null)) { await manager.InitializationComplete; Assert.IsTrue(manager.AddOns.Count > 0); } }
// ------------------------------------------ // HTTP Management // ------------------------------------------ public override void HandleBODY(string device, string body, string token) { base.HandleBODY(device, body, token); if ("speech".Equals(token)) { AddOnManager.GetInstance().BeforeHandleVoice(body, true); } }
public MainWindowContext(AddOnManager addOnManager) { showPrereleaseVersions = App.ShowPrereleaseVersions; AddOnManager = addOnManager; SortedClients = addOnManager.WorldOfWarcraftInstallation.Clients.ActiveOrderBy(client => client.FlavorName); firstSortedClient = SortedClients.ActiveFirstOrDefault(); firstSortedClient.PropertyChanged += FirstSortedClientPropertyChangedHandler; selectedClient = firstSortedClient.Value; }
// ------------------------------------------ // Audio Management // ------------------------------------------ public override void BeforeSpeechRecognition(string device, string text, double confidence, XPathNavigator xnav, string grammar, Stream stream, IDictionary <string, string> options) { base.BeforeSpeechRecognition(device, text, confidence, xnav, grammar, stream, options); var tts = xnav.SelectSingleNode("/SML/action/@tts"); if (tts != null) { AddOnManager.GetInstance().BeforeHandleVoice(tts.Value, true); } }
private void OnTriggerEnter(Collider other) { if (other.CompareTag("Player")) { if (gameObject.CompareTag("Point")) { AddOnManager.collectedPoint(); GameManager.IndexOfCoinsCollected.Add(transform.parent.gameObject.GetComponent <Points>().Index); Destroy(transform.parent.gameObject); } } }
public override void BeforeHTTPRequest(string qs, NameValueCollection parameters, IDictionary files, StreamWriter writer) { base.BeforeHTTPRequest(qs, parameters, files, writer); // Text to Speech String tts = parameters.Get("tts"); if (tts != null) { AddOnManager.GetInstance().BeforeHandleVoice(tts, parameters.Get("sync") != null); } }
// ------------------------------------------- // SPEECH // ------------------------------------------- protected bool Confidence(String device, RecognitionResult rr, XPathNavigator xnav, double threashold) { double confidence = rr.Confidence; // Override Engine threashold by grammar XPathNavigator level = xnav.SelectSingleNode("/SML/action/@threashold"); if (level != null) { Log("Override confidence: " + level.Value); threashold = level.ValueAsDouble; } // Search for bot name double match = 0; double bot = ConfigManager.GetInstance().Find("bot.confidence", 0.0); string name = ConfigManager.GetInstance().Find("bot.name", "SARAH").ToUpper(); foreach (var word in rr.Words) { if (!name.Equals(word.Text.ToUpper())) { continue; } if (word.Confidence < threashold + bot) // Check the bot name with threashold confidence { Log("REJECTED " + name + ": " + word.Confidence + " < " + (threashold + bot) + " (" + rr.Text + ")"); return(false); } match = word.Confidence; break; } // Must have bot name in sentence var grammar = GrammarManager.GetInstance().FindGrammar(rr.Grammar.Name); if (match == 0 && grammar.HasName && !AddOnManager.GetInstance().IsEngaged(device)) { return(false); } // Check full sentence if (confidence < threashold) { Log("REJECTED: " + confidence + " (" + match + ") " + " < " + threashold + " (" + rr.Text + ")"); return(false); } // Speech recognized Log("RECOGNIZED: " + confidence + " (" + match + ") " + " > " + threashold + " (" + rr.Text + ")"); return(true); }
// ------------------------------------------ // AUDIO // ------------------------------------------ private void StartAudioStream() { IReadOnlyList <AudioBeam> audioBeamList = Sensor.AudioSource.AudioBeams; var beam = audioBeamList[0]; var stream = beam.OpenInputStream(); var audio = new KinectAudioStream(stream); double confidence = ConfigManager.GetInstance().Find("kinect_v2.speech.confidence", 0.6); // Let the convertStream know speech is going active audio.SpeechActive = true; // Build Speech Engine AddOnManager.GetInstance().AddAudioSource(Name, audio, "Kinect", null, confidence); }
private async Task RepaintAsync(TimeSpan dueTime, TimeSpan interval, CancellationToken token) { // Initial wait time before we begin the periodic loop. if (dueTime > TimeSpan.Zero) { await Task.Delay(dueTime, token); } // Repeat this loop until cancelled. while (!token.IsCancellationRequested) { // Timestamp data RepaintWatch.Again(); // Do Job try { // Init once InitRepaint(); // Copy frame ColorFrame color = (ColorFrame)Descriptor.Find(typeof(ColorFrame)); if (color != null) { Buffer.BlockCopy(color.Pixels, 0, drawer, 0, color.Pixels.Length); // Buffer.BlockCopy(Infrared, 0, drawer, 0, Infrared.Length); // Ask repaint to addons AddOnManager.GetInstance().RepaintColorFrame(Name, drawer, color.Width, color.Height); } // Resize scale.Bytes = drawer; var resize = scale.Resize(w, h, Emgu.CV.CvEnum.Inter.Cubic); bitmap.WritePixels(rect, resize.Bytes, stride, 0); // Ticks this.Repaint.Text = "Repaint: " + RepaintWatch.Average(); } catch (Exception ex) { Error(ex); } RepaintWatch.Stop(); // Wait to repeat again. try { if (interval > TimeSpan.Zero) { await Task.Delay(interval, token); } } catch (ThreadInterruptedException) { break; } } }
public override void Setup() { base.Setup(); buffer = new StreamBuffer(); int port = ConfigManager.GetInstance().Find("rtp.port", 7887); client = new RTPClient(port, buffer); client.StartClient(); double confidence = ConfigManager.GetInstance().Find("rtp.confidence", 0.6); AddOnManager.GetInstance().AddAudioSource("RTP", buffer, "RTP", null, confidence); }
private void InitFrames(DepthImageFrame depthFrame, ColorImageFrame colorFrame, SkeletonFrame skeletonFrame) { if (init) { return; } init = true; // Color Frame Color = new ColorFrame(); Color.Width = colorFrame.Width; Color.Height = colorFrame.Height; Color.Pixels = new byte[colorFrame.PixelDataLength]; Color.Stamp = new Timestamp(); Color.Fps = FPS; AddOnManager.GetInstance().InitFrame(Name, Color); Log(Color.ToString()); ColorFormat = colorFrame.Format; // Depth Frame Depth = new DepthFrame(); Depth.Width = depthFrame.Width; Depth.Height = depthFrame.Height; Depth.Pixelss = new short[depthFrame.PixelDataLength]; Depth.Stamp = new Timestamp(); AddOnManager.GetInstance().InitFrame(Name, Depth); Log(Depth.ToString()); var dueTime = TimeSpan.FromMilliseconds(200); var interval = TimeSpan.FromMilliseconds(ConfigManager.GetInstance().Find("kinect_v1.motion.ms", 100)); Task = new MotionTask(dueTime, interval); Task.Device = ""; Task.AddFrame(Depth); Task.Start(); // Skeleton Frame Skeletons = new BodyFrame(); Skeletons.Width = colorFrame.Width; Skeletons.Height = colorFrame.Height; Skeletons.RawData = new Skeleton[6]; Skeletons.Bodies = new List <NBody>(6); Skeletons.Stamp = new Timestamp(); AddOnManager.GetInstance().InitFrame(Name, Skeletons); Log(Skeletons.ToString()); }
protected override void DoTask() { var names = Helper.Recognize(); if (names == null) { return; } for (var i = 0; i < names.Length; i++) { if (names[i] == null) { continue; } // Try to match a body var rect1 = Helper.GetArea(names[i]); foreach (var body in Body.Cache(cache)) { if (body == null) { continue; } var head = body.GetJoint(NJointType.Head); var rect2 = head.Area; if (body.Name != null && names[i] == FaceHelper.UNKNOWN) { continue; } if (rect1.IntersectsWith(rect2) || rect1.Contains(rect2) || rect2.Contains(rect1)) { body.Name = names[i]; } } AddOnManager.GetInstance().HandleProfile(Device, "face", names[i]); } var image = Helper.RecognizeThumbs[0]; if (thumb != null && image != null) { Buffer.BlockCopy(image.Bytes, 0, thumb, 0, thumb.Length); } }
private void Grid_MouseUp(object sender, MouseButtonEventArgs e) { // Release the mouse capture and stop tracking it. mouseDown = false; Wrapper.ReleaseMouseCapture(); // Hide the drag selection box. selectionBox.Visibility = Visibility.Collapsed; var rectBox = new System.Drawing.Rectangle( (int)Canvas.GetLeft(selectionBox), (int)Canvas.GetTop(selectionBox), (int)selectionBox.Width, (int)selectionBox.Height); var rectImg = new System.Drawing.Rectangle((int)(rectBox.X * ratio), (int)(rectBox.Y * ratio), (int)(rectBox.Width * ratio), (int)(rectBox.Height * ratio)); AddOnManager.GetInstance().HandleSelection(Name, rectImg); }
private void StartBodyStream() { // Init infr buffer BodyFrame frame = Body = new BodyFrame(); frame.Width = Color.Width; frame.Height = Color.Height; frame.RawData = new Body[6]; // FIXME frame.Bodies = new List <NBody>(6); frame.Stamp = new Timestamp(); AddOnManager.GetInstance().InitFrame(Name, frame); Log(frame.ToString()); // Start Watch BodyWatch = new StopwatchAvg(); }
public async Task AddOnStatesPersistAsync() { var blankAddOnKey = Guid.Parse("945fad13-7ec7-4149-9541-3852bfad0673"); var testStorageDirectory = GetTestStorageDirectory(); using (var manager = new AddOnManager(testStorageDirectory, null)) { await manager.InitializationComplete; Assert.IsTrue(await manager.AddOns.First(addOn => addOn.Key == blankAddOnKey).DownloadAsync()); } using (var manager = new AddOnManager(testStorageDirectory, null)) { await manager.InitializationComplete; var blankAddOn = manager.AddOns.First(addOn => addOn.Key == blankAddOnKey); Assert.IsTrue(blankAddOn.IsDownloaded); await blankAddOn.DeleteAsync(); } CleanTestStorageDirectory(testStorageDirectory); }
private void StartBodyIndexStream() { // Get frame description for the infr output var description = Sensor.BodyIndexFrameSource.FrameDescription; // Init infr buffer BodyIndexFrame frame = BodyIndex = new BodyIndexFrame(); frame.Width = description.Width; frame.Height = description.Height; frame.Pixels = new byte[description.LengthInPixels]; frame.Stamp = new Timestamp(); AddOnManager.GetInstance().InitFrame(Name, frame); Log(frame.ToString()); // Start Watch BodyIndexWatch = new StopwatchAvg(); }
async void Initialize(object state) { if (!singleInstance.IsFirstInstance) { await singleInstance.SendMessageAsync("showmainwindow"); Environment.Exit(0); } updateAvailableVersion = new Timer(UpdateAvailableVersion, null, TimeSpan.Zero, TimeSpan.FromDays(1)); try { worldOfWarcraftInstallation = new WorldOfWarcraftInstallation(synchronizationContext: synchronizationContext); } catch (WorldOfWarcraftInstallationUnavailableException) { await OnUiThreadAsync(() => MessageBox.Show("Dude and/or madame, you have World of Warcraft installed in an odd place or not at all. I can't manage add-ons for WoW without WoW!", "NOOOOOOOOPE", MessageBoxButton.OK, MessageBoxImage.Error)).ConfigureAwait(false); Environment.Exit(0); } addOnManager = new AddOnManager(await Utilities.GetCommonStorageDirectoryAsync().ConfigureAwait(false), worldOfWarcraftInstallation, synchronizationContext); addOnManager.AddOnAutomaticallyUpdated += AddOnManagerAddOnAutomaticallyUpdatedHandler; addOnManager.AddOnUpdateAvailable += AddOnManagerAddOnUpdateAvailableHandler; stateFile = new FileInfo(Path.Combine(addOnManager.StorageDirectory.FullName, "appState.json")); if (stateFile.Exists) { var appState = JsonConvert.DeserializeObject <AppState>(await File.ReadAllTextAsync(stateFile.FullName).ConfigureAwait(false)); MainWindowHeight = appState.MainWindowHeight; MainWindowLeft = appState.MainWindowLeft; MainWindowTop = appState.MainWindowTop; MainWindowWidth = appState.MainWindowWidth; ShowPrereleaseVersions = appState.ShowPrereleaseVersions; themeIsDark = appState.ThemeIsDark; themeIsHorde = appState.ThemeIsHorde; await OnUiThreadAsync(() => SetTheme()).ConfigureAwait(false); } await CreateMainWindow(openMinimized : Environment.GetCommandLineArgs().Contains("-startMinimized", StringComparer.OrdinalIgnoreCase)).ConfigureAwait(false); }
protected void http_RequestReceived(object sender, HttpRequestEventArgs e) { Log("Request received: " + e.Request.Url.AbsoluteUri); var qs = e.Request.Url.Query; var parameters = e.Request.Params; var files = new Dictionary <string, string>(); var temp = ConfigManager.GetInstance().Find("http.local.temp", "AddOns/http/temp/"); // Dump all files in a temporary directory foreach (string key in e.Request.Files.Keys) { var file = e.Request.Files.Get(key); if (file == null) { continue; } using (var reader = new BinaryReader(file.InputStream)) { var data = reader.ReadBytes(file.ContentLength); var path = temp + file.FileName; if (File.Exists(path)) { File.Delete(path); } File.WriteAllBytes(path, data); files.Add(key, path); } } // Fake response using (var writer = new StreamWriter(e.Response.OutputStream)) { // Handle custom request AddOnManager.GetInstance().BeforeHTTPRequest(qs, parameters, files, writer); // Write to stream writer.Write(" "); writer.Flush(); writer.Close(); } AddOnManager.GetInstance().AfterHTTPRequest(qs, parameters, files); }
private void StartColorStream() { // Get frame description for the infr output var description = Sensor.ColorFrameSource.FrameDescription; // Init infr buffer ColorFrame frame = Color = new ColorFrame(); frame.Width = description.Width; frame.Height = description.Height; frame.Pixels = new byte[description.LengthInPixels * 4]; frame.Stamp = new Timestamp(); frame.Fps = 15; AddOnManager.GetInstance().InitFrame(Name, frame); Log(frame.ToString()); // Start Watch ColorWatch = new StopwatchAvg(); }