public void Load(String xmlFile)
{
var reader = new XmlTextReader(xmlFile);
Gesture gesture = null;
while (reader.Read())
{
switch (reader.NodeType)
{
case XmlNodeType.Element:
if (reader.Name == "gestures")
{
var timeout = reader.GetAttribute("gestureResetTimeout");
gestureResetTimeout = timeout != null?Convert.ToInt32(timeout) : gestureResetTimeout;
}
if (reader.Name == "gesture")
{
gesture = Gesture.Parse(reader);
gestures.Add(gesture);
WSRConfig.GetInstance().logInfo("GESTURE", "Loading: " + gesture.Description);
}
if (reader.Name == "component" && gesture != null)
{
GestureComponent component = GestureComponent.Parse(reader);
gesture.Components.Add(component);
WSRConfig.GetInstance().logDebug("GESTURE", "Component: " + component.Log(gesture.Description));
}
break;
}
}
}
public bool Evaluate(Skeleton sd, DateTime currentTime)
{
// Check recognition timeout
if (IsExecuting)
{
TimeSpan executiontime = currentTime - beginExecutionTime;
if (executiontime.TotalMilliseconds > gesture.MaximumExecutionTime && gesture.MaximumExecutionTime > 0)
{
Reset();
return(false);
}
}
// Check each component of the gesture
foreach (var component in ComponentStates)
{
if (component.Evaluate(sd))
{
WSRConfig.GetInstance().logDebug("GESTURE", "Gesture Component: " + gesture.Description);
}
}
// Check gesture is completed
var inflightCount = 0;
var completeCount = 0;
foreach (var component in ComponentStates)
{
if (component.IsBegin)
{
inflightCount++;
}
if (component.IsEnding)
{
completeCount++;
}
}
if (completeCount >= ComponentStates.Count && IsExecuting)
{
WSRConfig.GetInstance().logDebug("GESTURE", "Gesture complete: " + gesture.Description);
WSRConfig.GetInstance().logDebug("GESTURE", ">>>> RESET <<<<");
Reset();
return(true);
}
// Some components match
if (inflightCount >= ComponentStates.Count)
{
if (!IsExecuting)
{
WSRConfig.GetInstance().logDebug("GESTURE", "Has Transitioned To In Flight State: " + gesture.Description);
IsExecuting = true;
beginExecutionTime = DateTime.Now;
return(false);
}
}
return(false);
}
protected void InitSpeechEngine(bool def)
{
try {
WSRConfig cfg = WSRConfig.GetInstance();
WSRSpeechManager manager = WSRSpeechManager.GetInstance();
// File
manager.InitEngines();
// Default
if (def)
{
manager.AddDefaultEngine("Default", cfg.language, cfg.confidence);
}
// RTP
if (rtpClient == null)
{
return;
}
var format = new SpeechAudioFormatInfo(16000, AudioBitsPerSample.Sixteen, AudioChannel.Stereo);
manager.AddEngine("RTP", cfg.language, cfg.confidence, rtpClient.AudioStream, format);
}
catch (Exception ex) {
WSRConfig.GetInstance().logError("ENGINE", "InitEngines: " + ex.Message);
}
}
public void SetContext(String context)
{
if (context == null)
{
return;
}
logInfo("GRAMMAR", "Context: " + context);
if ("default".Equals(context))
{
SetContext(WSRConfig.GetInstance().context);
tmpContext = null;
return;
}
bool all = "all".Equals(context);
foreach (WSRSpeecGrammar g in Cache.Values)
{
if (g.Name == "dictation")
{
continue;
}
g.Enabled = all || context.Equals(g.Name);
logDebug("CONTEXT", g.Name + " = " + g.Enabled);
}
}
// ==========================================
// CONSTRUCTOR
// ==========================================
public WSRSpeechEngine(String name, String language, double confidence)
{
this.Name = name;
this.Confidence = confidence;
this.cfg = WSRConfig.GetInstance();
this.engine = new SpeechRecognitionEngine(new System.Globalization.CultureInfo(language));
}
static void Main(string[] args)
{
var config = WSRConfig.GetInstance();
if (!config.Parse(args).Validate())
{
return;
}
try {
// Setup logging
config.SetupLogging();
// Start Process
Application.EnableVisualStyles();
Application.SetCompatibleTextRenderingDefault(false);
// Buid WSRMicro or Kinect
config.Start();
// Build System Tray
using (WSRTrayMenu tray = new WSRTrayMenu()) {
tray.Display();
Application.Run(); // Make sure the application runs!
}
}
catch (Exception ex) {
config.logError("CATCH_ALL", ex);
}
}
// ==========================================
// CONSTRUCTOR
// ==========================================
public WSRSpeechEngine(String name, String language, double confidence)
{
this.Name = name;
this.Confidence = confidence;
this.cfg = WSRConfig.GetInstance();
this.engine = new SpeechRecognitionEngine(new System.Globalization.CultureInfo(language));
}
public void UpdatePitch(double pitch)
{
bool hasChange = Profiles.RemoveAll(ProfileTimout) > 0;
// Search for WSRProfile
var delta = null != Current?Math.Abs(Current.Pitch - pitch) : pitch;
foreach (WSRProfile profile in Profiles)
{
var d = Math.Abs(profile.Pitch - pitch);
if (delta < d)
{
continue;
}
Current = profile;
delta = d;
}
// Create new WSRProfile
if (null == Current || (Current.Pitch > 0 && delta > WSRConfig.GetInstance().PitchDelta))
{
Current = new WSRProfile();
Profiles.Add(Current);
}
// Reset TimeStamp
Current.Timestamp = DateTime.Now;
Current.Pitch = pitch;
// Trigger
ProfileChange(hasChange);
}
/// <summary>
/// Handles the receiving of UDP packets from the RTP stream
/// </summary>
/// <param name="ar">Contains packet data</param>
private void ReceiveCallback()
{
WSRConfig.GetInstance().logInfo("RTPClient", "ReceiveCallback");
// Begin looking for the next packet
while (listening)
{
// Receive packet
try {
byte[] packet = client.Receive(ref endPoint);
// Decode the header of the packet
int version = GetRTPHeaderValue(packet, 0, 1);
int padding = GetRTPHeaderValue(packet, 2, 2);
int extension = GetRTPHeaderValue(packet, 3, 3);
int csrcCount = GetRTPHeaderValue(packet, 4, 7);
int marker = GetRTPHeaderValue(packet, 8, 8);
int payloadType = GetRTPHeaderValue(packet, 9, 15);
int sequenceNum = GetRTPHeaderValue(packet, 16, 31);
int timestamp = GetRTPHeaderValue(packet, 32, 63);
int ssrcId = GetRTPHeaderValue(packet, 64, 95);
if (writeHeaderToConsole)
{
WSRConfig.GetInstance().logDebug("RTPClient",
version + ", " + padding + ", " + extension + ", " + csrcCount + ", " + marker + ", " + payloadType + ", " + sequenceNum + ", " + timestamp + ", " + ssrcId);
}
// Write the packet to the audio stream
audioStream.Write(packet, 12, packet.Length - 12);
}
catch (SocketException) { break; } // exit the while loop
}
}
public void StartHttpServer()
{
int port = WSRConfig.GetInstance().loopback;
IPAddress address = GetIpAddress();
// 192.168.0.x
if (address != null)
{
try {
http = new HttpServer();
http.EndPoint = new IPEndPoint(address, port);
http.Start();
http.RequestReceived += this.http_RequestReceived;
WSRConfig.GetInstance().logInfo("INIT", "Starting Server: http://" + http.EndPoint + "/");
}
catch (Exception ex) {
http = null;
WSRConfig.GetInstance().logInfo("HTTP", "Exception: " + ex.Message);
}
}
// Localhost
try {
httpLocal = new HttpServer();
httpLocal.EndPoint = new IPEndPoint(IPAddress.Loopback, port);
httpLocal.RequestReceived += this.http_RequestReceived;
httpLocal.Start();
WSRConfig.GetInstance().logInfo("INIT", "Starting Server: http://" + httpLocal.EndPoint + "/");
}
catch (Exception ex) {
httpLocal = null;
WSRConfig.GetInstance().logInfo("HTTP", "Exception: " + ex.Message);
}
}
public void SendUpoad(string url, string path)
{
if (url == null)
{
return;
}
if (path == null)
{
SendRequest(url); return;
}
if (working)
{
return;
}
working = true;
url = CleanURL(url);
WSRConfig.GetInstance().logInfo("HTTP", "Build UploadRequest: " + url);
WebClient client = new WebClient();
client.Headers.Add("user-agent", "S.A.R.A.H. (Self Actuated Residential Automated Habitat)");
try {
byte[] responseArray = client.UploadFile(url, path);
String response = System.Text.Encoding.ASCII.GetString(responseArray);
WSRSpeakerManager.GetInstance().Speak(response, false);
}
catch (Exception ex) {
WSRConfig.GetInstance().logInfo("HTTP", "Exception: " + ex.Message);
}
working = false;
}
private async Task ColorAsync(TimeSpan dueTime, TimeSpan interval, CancellationToken token)
{
if (interval.TotalMilliseconds == 0)
{
return;
}
Stopwatch ColorTH = new Stopwatch();
ColorTH.Start();
// Initial wait time before we begin the periodic loop.
if (dueTime > TimeSpan.Zero)
{
await Task.Delay(dueTime, token);
}
DateTime LocalTimestamp = Timestamp;
WSRColor color = new WSRColor();
// Repeat this loop until cancelled.
while (!token.IsCancellationRequested)
{
// Skip already work with given data
if (Timestamp == LocalTimestamp)
{
await Task.Delay(interval, token);
continue;
}
// Timestamp data
LocalTimestamp = Timestamp;
ColorWatch.Again();
// Do Job
try {
CopyColorData = true;
var rgb = color.GetMostProminentColor(ColorData);
if (RGB == null || rgb.r > 50 && rgb.g > 50 && rgb.b > 50)
{
RGB = rgb;
}
if (WSRConfig.GetInstance().ColorTH.Milliseconds > 0 && ColorTH.Elapsed > WSRConfig.GetInstance().ColorTH)
{
WSRHttpManager.GetInstance().SendRequest("http://127.0.01:8080/sarah/hue?r=" + RGB.r + "&g=" + RGB.g + "&b=" + RGB.b);
ColorTH.Restart();
}
}
catch (Exception ex) {
WSRConfig.GetInstance().logError("COLOR", ex);
}
ColorWatch.Stop();
// Wait to repeat again.
if (interval > TimeSpan.Zero)
{
await Task.Delay(interval, token);
}
}
}
// Constructor
public WSRSpeakerManager()
{
// Enumerate all Speaker
if ("all" == WSRConfig.GetInstance().Speakers)
{
int waveOutDevices = WaveOut.DeviceCount;
for (int n = 0; n < waveOutDevices; n++)
{
var cap = WaveOut.GetCapabilities(n);
WSRConfig.GetInstance().logInfo("TTS", "Add Speaker Device: " + n
+ " Product: " + cap.ProductName
+ " Channels: " + cap.Channels
+ " Playback: " + cap.SupportsPlaybackRateControl);
speakers.Add(new WSRSpeaker(n));
}
return;
}
else
{
// Enumerate declared Speaker
foreach (var spkr in WSRConfig.GetInstance().Speakers.Split(','))
{
var idx = int.Parse(spkr);
var cap = WaveOut.GetCapabilities(idx);
WSRConfig.GetInstance().logInfo("TTS", "Add Speaker Device: " + idx
+ " Product: " + cap.ProductName
+ " Channels: " + cap.Channels
+ " Playback: " + cap.SupportsPlaybackRateControl);
speakers.Add(new WSRSpeaker(idx));
}
}
}
// ------------------------------------------
// CONSTRUCTOR
// ------------------------------------------
public WSRSpeaker(int device)
{
this.device = device;
this.WaveOutSpeech = new WaveOut(WaveCallbackInfo.FunctionCallback());
this.WaveOutSpeech.DeviceNumber = device;
this.synthesizer = new SpeechSynthesizer();
this.synthesizer.SpeakCompleted += new EventHandler <SpeakCompletedEventArgs>(synthesizer_SpeakCompleted);
// Enumerate Voices
foreach (InstalledVoice voice in synthesizer.GetInstalledVoices())
{
VoiceInfo info = voice.VoiceInfo;
WSRConfig.GetInstance().logInfo("TTS", "[" + device + "]" + "Name: " + info.Name + " Culture: " + info.Culture);
}
// Select Voice
String v = WSRConfig.GetInstance().voice;
if (v != null && v.Trim() != "")
{
WSRConfig.GetInstance().logInfo("TTS", "[" + device + "]" + "Select voice: " + v);
synthesizer.SelectVoice(v);
}
}
public void LoadXML(String file, String name)
{
WSRConfig cfg = WSRConfig.GetInstance();
WSRSpeecGrammar grammar = null;
if (Cache.TryGetValue(name, out grammar))
{
if (grammar.LastModified == File.GetLastWriteTime(file))
{
logDebug("GRAMMAR", "Ignoring: " + name + " (no changes)");
return;
}
}
try {
// Load the XML
logInfo("GRAMMAR", "Load file: " + name + " : " + file);
String xml = File.ReadAllText(file, Encoding.UTF8);
xml = Regex.Replace(xml, "([^/])SARAH", "$1" + cfg.Name, RegexOptions.IgnoreCase);
// Check regexp language
if (!Regex.IsMatch(xml, "xml:lang=\"" + cfg.language + "\"", RegexOptions.IgnoreCase))
{
logInfo("GRAMMAR", "Ignoring : " + name + " (" + cfg.language + ")");
return;
}
// New grammar
if (null == grammar)
{
grammar = new WSRSpeecGrammar();
Cache.Add(name, grammar); // Add to cache
}
// Build grammar
grammar.XML = xml;
grammar.Name = name;
grammar.Path = file;
grammar.LastModified = File.GetLastWriteTime(file);
// Check contexte
grammar.Enabled = true;
if ((file.IndexOf("lazy") >= 0) || Regex.IsMatch(xml, "root=\"lazy\\w+\"", RegexOptions.IgnoreCase))
{
grammar.Enabled = false;
}
// Add to context if there is no context
if (!WSRConfig.GetInstance().HasContext() && grammar.Enabled && !WSRConfig.GetInstance().context.Contains(name))
{
WSRConfig.GetInstance().context.Add(name);
logInfo("GRAMMAR", "Add to context list: " + name);
}
}
catch (Exception ex) {
cfg.logError("GRAMMAR", ex);
}
}
// ------------------------------------------
// Load gesture
// ------------------------------------------
public void Load()
{
foreach (string directory in WSRConfig.GetInstance().directories)
{
DirectoryInfo d = new DirectoryInfo(directory);
LoadGestures(d);
}
}
public void RunApp(String processName, String param)
{
try {
Process.Start(processName, param);
} catch (Exception ex) {
WSRConfig.GetInstance().logError("KEYBOARD", ex);
}
}
public static WSRConfig GetInstance()
{
if (config == null)
{
config = new WSRConfig();
}
return(config);
}
/// <summary>
/// Tells the UDP client to stop listening for packets.
/// </summary>
public void StopClient()
{
// Set the boolean to false to stop the asynchronous packet receiving
listening = false;
listenerThread.Interrupt();
try { client.Close(); } catch (SocketException) { }
WSRConfig.GetInstance().logInfo("RTPClient", "Stopped listening on port " + port);
}
// Handles the MouseClick event of the ni control.
void ni_MouseClick(object sender, MouseEventArgs e)
{
// Handle mouse button clicks.
if (e.Button == MouseButtons.Left)
{
// Start Windows Explorer.
Process.Start("explorer", WSRConfig.GetInstance().getDirectory());
}
}
public void ActivateApp(string processName)
{
// Activate the first application we find with this name
Process[] p = Process.GetProcessesByName(processName);
if (p.Length > 0)
{
SetForegroundWindow(p[0].MainWindowHandle);
WSRConfig.GetInstance().logInfo("KEYBOARD", "Activate " + p[0].ProcessName);
}
}
protected void DisposeRTPClient()
{
WSRConfig.GetInstance().logError("RTPCLIENT", "Stop RTPClient");
if (rtpClient != null)
{
rtpClient.StopClient();
rtpClient.Dispose();
}
WSRConfig.GetInstance().logError("RTPCLIENT", "Stop RTPClient ... Done");
}
private async Task RepaintAsync(TimeSpan dueTime, TimeSpan interval, CancellationToken token)
{
// Initial wait time before we begin the periodic loop.
if (dueTime > TimeSpan.Zero)
{
await Task.Delay(dueTime, token);
}
int ColorW = Sensor.ColorW;
int ColorH = Sensor.ColorH;
// Repeat this loop until cancelled.
while (!token.IsCancellationRequested)
{
// Timestamp data
RepaintWatch.Again();
// Do Job
try {
// Draw Profile
DrawProfile();
// Draw Metrics
DrawMetrics();
if (!WSRConfig.GetInstance().SpeechOnly)
{
// Image color
Sensor.CopyColorData = true;
Bitmap bitmap = WSRKinectSensor.ToBitmap(Sensor.ColorData, ColorW, ColorH, System.Drawing.Imaging.PixelFormat.Format32bppRgb);
Image <Bgr, Byte> image = new Image <Bgr, Byte>(bitmap);
DrawFaces(image);
DrawJoints(image);
DrawFace(image);
DrawDepth(image);
this.Image.Source = ToBitmapSource(image);
// Prominent Color
DrawProminentColor();
}
}
catch (Exception ex) {
WSRConfig.GetInstance().logError("CAMERA", ex);
}
RepaintWatch.Stop();
// Wait to repeat again.
if (interval > TimeSpan.Zero)
{
await Task.Delay(interval, token);
}
}
}
/// <summary>
/// RTP Client for receiving an RTP stream containing a WAVE audio stream
/// </summary>
/// <param name="port">The port to listen on</param>
public RTPClient(int port)
{
WSRConfig.GetInstance().logInfo("RTPClient", "Loading...");
this.port = port;
// Initialize the audio stream that will hold the data
audioStream = new Streamer(AUDIO_BUFFER_SIZE);
WSRConfig.GetInstance().logInfo("RTPClient", "Done");
}
public void InitEngines()
{
Engines.Clear();
WSRConfig cfg = WSRConfig.GetInstance();
// File
WSRSpeechEngine fileEngine = new WSRSpeechEngine("File", cfg.language, cfg.confidence);
fileEngine.LoadGrammar();
fileEngine.Init();
Engines.Add("File", fileEngine);
}
public void SetContextTimeout()
{
if (ctxTimer != null)
{
return;
}
logInfo("CONTEXT", "Start context timeout: " + WSRConfig.GetInstance().ctxTimeout);
ctxTimer = new System.Timers.Timer();
ctxTimer.Interval = WSRConfig.GetInstance().ctxTimeout;
ctxTimer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
ctxTimer.Enabled = true;
ctxTimer.Start();
}
private void InitColorBitmap()
{
WSRConfig cfg = WSRConfig.GetInstance();
if (cfg.WSSmooth)
{
filter1 = new DepthFilteredSmoothing();
}
if (cfg.WSAverage)
{
filter2 = new DepthAveragedSmoothing();
}
}
/// <summary>
/// Creates a connection to the RTP stream
/// </summary>
public void StartClient()
{
// Create new UDP client. The IP end point tells us which IP is sending the data
client = new UdpClient(port);
endPoint = new IPEndPoint(IPAddress.Broadcast, port);
WSRConfig.GetInstance().logInfo("RTPClient", "Listening for packets on port " + port + "...");
listening = true;
listenerThread = new Thread(ReceiveCallback);
listenerThread.Name = "UDP Thread";
listenerThread.Start();
}
/**
* Can be called on RaspberryPi using ffmpeg
* ffmpeg -ac 1 -f alsa -i hw:1,0 -ar 16000 -acodec pcm_s16le -f rtp rtp://192.168.0.8:7887
* avconv -f alsa -ac 1 -i hw:0,0 -acodec mp2 -b 64k -f rtp rtp://{IP of your laptop}:1234
*
* Or using Kinect on windows
* ffmpeg -f dshow -i audio="Réseau de microphones (Kinect U" -ar 16000 -acodec pcm_s16le -f rtp rtp://127.0.0.1:7887
*/
protected void StartRTPClient()
{
var port = WSRConfig.GetInstance().rtpport;
if (port < 0)
{
return;
}
WSRConfig.GetInstance().logInfo("RTPCLIENT", "Start RTPClient: " + port);
rtpClient = new RTPClient(port);
rtpClient.StartClient();
}
public bool Speak(String tts, bool async)
{
// Run Async
if (async)
{
return(SpeakAsync(tts));
}
// Compute name
var name = WSRProfileManager.GetInstance().CurrentName();
tts = Regex.Replace(tts, "\\[name\\]", name, RegexOptions.IgnoreCase);
WSRConfig.GetInstance().logInfo("TTS", "Try speaking ... " + tts);
lock (Lock) {
// Prevent multiple Speaking
if (Speaking || working)
{
WSRConfig.GetInstance().logInfo("TTS", "Already speaking ... enqueue");
queue.Enqueue(tts); return(false);
}
working = true;
}
// Run all speech in a list of Task
List <Task> latest = new List <Task>();
SpeechBuffer += tts + " ";
foreach (var speaker in speakers)
{
latest.Add(Task.Factory.StartNew(() => speaker.Speak(tts)));
}
// Wait for end of Tasks
foreach (var task in latest)
{
task.Wait();
}
latest.Clear();
lock (Lock) { working = false; }
WSRConfig.GetInstance().logInfo("TTS", "End speaking");
// Dequeue next (always async ...)
if (queue.Count > 0)
{
Speak(queue.Dequeue(), true);
}
return(true);
}
private async Task FaceRecognitionAsync(TimeSpan dueTime, TimeSpan interval, CancellationToken token)
{
if (interval.TotalMilliseconds == 0)
{
return;
}
TimeSpan threshold = WSRConfig.GetInstance().FaceTH;
// Initial wait time before we begin the periodic loop.
if (dueTime > TimeSpan.Zero)
{
await Task.Delay(dueTime, token);
}
DateTime LocalTimestamp = Timestamp;
// Repeat this loop until cancelled.
while (!token.IsCancellationRequested)
{
// Skip already work with given data
if (Timestamp == LocalTimestamp)
{
await Task.Delay(interval, token);
continue;
}
// Timestamp data
LocalTimestamp = Timestamp;
FaceRecoWatch.Again();
// Do Job
try {
var names = FaceManager.Recognize();
if (null != names)
{
WSRProfileManager.GetInstance().UpdateFace(names);
}
}
catch (Exception ex) {
WSRConfig.GetInstance().logError("FACE", ex);
}
FaceRecoWatch.Stop();
// Wait to repeat again.
if (interval > TimeSpan.Zero)
{
await Task.Delay(interval, token);
}
}
}
public static WSRConfig GetInstance()
{
if (config == null) {
config = new WSRConfig();
}
return config;
}