/// <summary>
/// Print (first n) messages from stream.
/// </summary>
/// <param name="stream">Stream name.</param>
/// <param name="store">Store name.</param>
/// <param name="path">Store path.</param>
/// <param name="number">Number of messages to display.</param>
/// <returns>Success flag.</returns>
internal static int DisplayStreamMessages(string stream, string store, string path, int number)
{
Console.WriteLine($"Stream Messages (stream={stream}, store={store}, path={path}, number={number})");
using (var pipeline = Pipeline.Create())
{
var count = 0;
var data = PsiStore.Open(pipeline, store, Path.GetFullPath(path));
data.OpenDynamicStream(stream).Do((m, e) =>
{
if (count++ < number)
{
PrintMessage(m, e);
}
});
pipeline.RunAsync();
while (count < number)
{
Thread.Sleep(100);
}
}
return(0);
}
public async Task SessionCreateDerivedPartitionCancellation()
{
var dataset = new Dataset();
var session = dataset.CreateSession();
// generate a test store
var originalStoreName = "OriginalStore";
GenerateTestStore(originalStoreName, StorePath);
// add a partition
var partition0 = session.AddPsiStorePartition(originalStoreName, StorePath, "Partition_0");
Assert.AreEqual(1, session.Partitions.Count);
Assert.AreEqual("Partition_0", session.Partitions[0].Name);
int multiplier = 7;
try
{
// create a cancellation token source that will automatically request cancellation after 500 ms
var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(500));
// create a derived partition which contains the values from the original stream multiplied by a multiplier
await session.CreateDerivedPsiPartitionAsync(
(pipeline, importer, exporter, parameter) =>
{
var inputStream = importer.OpenStream <int>("Root");
var derivedStream = inputStream.Sample(TimeSpan.FromMinutes(1), RelativeTimeInterval.Infinite).Select(x => x *parameter).Write("DerivedStream", exporter);
// add a dummy source and propose a long time interval so that the operation will block (and eventually be canceled)
var generator = Generators.Repeat(pipeline, 0, int.MaxValue, TimeSpan.FromMilliseconds(1000));
var replayTimeInterval = TimeInterval.LeftBounded(importer.MessageOriginatingTimeInterval.Left);
pipeline.ProposeReplayTime(replayTimeInterval);
},
multiplier,
"Partition_1",
false,
"DerivedStore",
StorePath,
replayDescriptor : null,
deliveryPolicy : null,
enableDiagnostics : false,
progress : null,
cancellationToken : cts.Token);
}
catch (OperationCanceledException)
{
// should NOT have created a new partition (but original partition should be intact)
Assert.AreEqual(1, session.Partitions.Count);
Assert.IsTrue(PsiStore.Exists("OriginalStore", StorePath));
Assert.IsFalse(PsiStore.Exists("DerivedStore", StorePath));
// verify original partition metadata
var originalPartition = session.Partitions[0] as Partition <PsiStoreStreamReader>;
Assert.AreEqual("Partition_0", originalPartition.Name);
Assert.AreEqual("OriginalStore", originalPartition.StoreName);
Assert.AreEqual($"{StorePath}{Path.DirectorySeparatorChar}OriginalStore.0000", originalPartition.StorePath);
throw;
}
}
public async Task SessionCreateDerivedPartition()
{
var dataset = new Dataset();
var session = dataset.CreateSession();
// generate a test store
var originalStoreName = "OriginalStore";
GenerateTestStore(originalStoreName, StorePath);
// add a partition
var partition0 = session.AddPsiStorePartition(originalStoreName, StorePath, "Partition_0");
Assert.AreEqual(1, session.Partitions.Count);
Assert.AreEqual("Partition_0", session.Partitions[0].Name);
int multiplier = 7;
// create a derived partition which contains the values from the original stream multiplied by a multiplier
await session.CreateDerivedPsiPartitionAsync(
(pipeline, importer, exporter, parameter) =>
{
var inputStream = importer.OpenStream <int>("Root");
var derivedStream = inputStream.Select(x => x *parameter).Write("DerivedStream", exporter);
},
multiplier,
"Partition_1",
false,
"DerivedStore",
StorePath);
// should have created a new store partition
Assert.AreEqual(2, session.Partitions.Count);
Assert.IsTrue(PsiStore.Exists("OriginalStore", StorePath));
Assert.IsTrue(PsiStore.Exists("DerivedStore", StorePath));
// verify partition metadata
var originalPartition = session.Partitions[0] as Partition <PsiStoreStreamReader>;
Assert.AreEqual("Partition_0", originalPartition.Name);
Assert.AreEqual("OriginalStore", originalPartition.StoreName);
Assert.AreEqual($"{StorePath}{Path.DirectorySeparatorChar}OriginalStore.0000", originalPartition.StorePath);
var derivedPartition = session.Partitions[1] as Partition <PsiStoreStreamReader>;
Assert.AreEqual("Partition_1", derivedPartition.Name);
Assert.AreEqual("DerivedStore", derivedPartition.StoreName);
Assert.AreEqual(StorePath, derivedPartition.StorePath);
// collections to capture stream values
var originalValues = new List <int>();
var originalTimes = new List <DateTime>();
var derivedValues = new List <int>();
var derivedTimes = new List <DateTime>();
// read stream values from the partitions
using (var pipeline = Pipeline.Create())
{
var originalPartitionImporter = PsiStore.Open(pipeline, originalPartition.StoreName, originalPartition.StorePath);
originalPartitionImporter.OpenStream <int>("Root").Do(
(i, e) =>
{
originalValues.Add(i);
originalTimes.Add(e.OriginatingTime);
});
var derivedPartitionImporter = PsiStore.Open(pipeline, derivedPartition.StoreName, derivedPartition.StorePath);
derivedPartitionImporter.OpenStream <int>("DerivedStream").Do(
(i, e) =>
{
derivedValues.Add(i);
derivedTimes.Add(e.OriginatingTime);
});
pipeline.Run();
}
// verify that we read the data
Assert.AreEqual(10, originalValues.Count);
Assert.AreEqual(10, originalTimes.Count);
Assert.AreEqual(10, derivedValues.Count);
Assert.AreEqual(10, derivedTimes.Count);
// verify values from both streams are what we expect
CollectionAssert.AreEqual(originalValues.Select(x => x * multiplier).ToList(), derivedValues);
CollectionAssert.AreEqual(originalTimes, derivedTimes);
}
/// <summary>
/// Builds and runs a speech recognition pipeline using the Azure speech recognizer. Requires a valid Cognitive Services
/// subscription key. See https://docs.microsoft.com/en-us/azure/cognitive-services/cognitive-services-apis-create-account.
/// </summary>
/// <remarks>
/// If you are getting a <see cref="System.InvalidOperationException"/> with the message 'AzureSpeechRecognizer returned
/// OnConversationError with error code: LoginFailed. Original error text: Transport error', this most likely is due to
/// an invalid subscription key. Please check your Azure portal at https://portal.azure.com and ensure that you have
/// added a subscription to the Azure Speech API on your account.
/// </remarks>
/// <param name="outputLogPath">The path under which to write log data.</param>
/// <param name="inputLogPath">The path from which to read audio input data.</param>
public static void RunAzureSpeech(string outputLogPath = null, string inputLogPath = null)
{
// Create the pipeline object.
using (Pipeline pipeline = Pipeline.Create())
{
// Use either live audio from the microphone or audio from a previously saved log
IProducer <AudioBuffer> audioInput = null;
if (inputLogPath != null)
{
// Open the MicrophoneAudio stream from the last saved log
var store = PsiStore.Open(pipeline, Program.AppName, inputLogPath);
audioInput = store.OpenStream <AudioBuffer>($"{Program.AppName}.MicrophoneAudio");
}
else
{
// Create the AudioCapture component to capture audio from the default device in 16 kHz 1-channel
// PCM format as required by both the voice activity detector and speech recognition components.
audioInput = new AudioCapture(pipeline, WaveFormat.Create16kHz1Channel16BitPcm());
}
// Perform voice activity detection using the voice activity detector component
var vad = new SystemVoiceActivityDetector(pipeline);
audioInput.PipeTo(vad);
// Create Azure speech recognizer component
var recognizer = new AzureSpeechRecognizer(pipeline, new AzureSpeechRecognizerConfiguration()
{
SubscriptionKey = Program.azureSubscriptionKey, Region = Program.azureRegion
});
// The input audio to the Azure speech recognizer needs to be annotated with a voice activity flag.
// This can be constructed by using the Psi Join() operator to combine the audio and VAD streams.
var annotatedAudio = audioInput.Join(vad);
// Subscribe the recognizer to the annotated audio
annotatedAudio.PipeTo(recognizer);
// Partial and final speech recognition results are posted on the same stream. Here
// we use Psi's Where() operator to filter out only the final recognition results.
var finalResults = recognizer.Out.Where(result => result.IsFinal);
// Print the recognized text of the final recognition result to the console.
finalResults.Do(result => Console.WriteLine(result.Text));
// Create a data store to log the data to if necessary. A data store is necessary
// only if output logging is enabled.
var dataStore = CreateDataStore(pipeline, outputLogPath);
// For disk logging only
if (dataStore != null)
{
// Log the microphone audio and recognition results
audioInput.Write($"{Program.AppName}.MicrophoneAudio", dataStore);
finalResults.Write($"{Program.AppName}.FinalRecognitionResults", dataStore);
vad.Write($"{Program.AppName}.VoiceActivity", dataStore);
}
// Register an event handler to catch pipeline errors
pipeline.PipelineExceptionNotHandled += Pipeline_PipelineException;
// Register an event handler to be notified when the pipeline completes
pipeline.PipelineCompleted += Pipeline_PipelineCompleted;
// Run the pipeline
pipeline.RunAsync();
// Azure speech transcribes speech to text
Console.WriteLine("Say anything");
Console.WriteLine("Press any key to exit...");
Console.ReadKey(true);
}
}
/// <summary>
/// Builds and runs a speech recognition pipeline using the .NET System.Speech recognizer and a set of fixed grammars.
/// </summary>
/// <param name="outputLogPath">The path under which to write log data.</param>
/// <param name="inputLogPath">The path from which to read audio input data.</param>
public static void RunSystemSpeech(string outputLogPath = null, string inputLogPath = null)
{
// Create the pipeline object.
using (Pipeline pipeline = Pipeline.Create())
{
// Use either live audio from the microphone or audio from a previously saved log
IProducer <AudioBuffer> audioInput = null;
if (inputLogPath != null)
{
// Open the MicrophoneAudio stream from the last saved log
var store = PsiStore.Open(pipeline, Program.AppName, inputLogPath);
audioInput = store.OpenStream <AudioBuffer>($"{Program.AppName}.MicrophoneAudio");
}
else
{
// Create the AudioCapture component to capture audio from the default device in 16 kHz 1-channel
// PCM format as required by both the voice activity detector and speech recognition components.
audioInput = new AudioCapture(pipeline, WaveFormat.Create16kHz1Channel16BitPcm());
}
// Create System.Speech recognizer component
var recognizer = new SystemSpeechRecognizer(
pipeline,
new SystemSpeechRecognizerConfiguration()
{
Language = "en-US",
Grammars = new GrammarInfo[]
{
new GrammarInfo()
{
Name = Program.AppName, FileName = "SampleGrammar.grxml"
},
},
});
// Subscribe the recognizer to the input audio
audioInput.PipeTo(recognizer);
// Partial and final speech recognition results are posted on the same stream. Here
// we use Psi's Where() operator to filter out only the final recognition results.
var finalResults = recognizer.Out.Where(result => result.IsFinal);
// Print the final recognition result to the console.
finalResults.Do(result =>
{
Console.WriteLine($"{result.Text} (confidence: {result.Confidence})");
});
// Create a data store to log the data to if necessary. A data store is necessary
// only if output logging is enabled.
var dataStore = CreateDataStore(pipeline, outputLogPath);
// For disk logging only
if (dataStore != null)
{
// Log the microphone audio and recognition results
audioInput.Write($"{Program.AppName}.MicrophoneAudio", dataStore);
finalResults.Write($"{Program.AppName}.FinalRecognitionResults", dataStore);
}
// Register an event handler to catch pipeline errors
pipeline.PipelineExceptionNotHandled += Pipeline_PipelineException;
// Register an event handler to be notified when the pipeline completes
pipeline.PipelineCompleted += Pipeline_PipelineCompleted;
// Run the pipeline
pipeline.RunAsync();
// The file SampleGrammar.grxml defines a grammar to transcribe numbers
Console.WriteLine("Say any number between 0 and 100");
Console.WriteLine("Press any key to exit...");
Console.ReadKey(true);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="RemoteImporter"/> class.
/// </summary>
/// <param name="pipeline">Pipeline to which to attach.</param>
/// <param name="host">Remote host name.</param>
/// <param name="port">TCP port on which to connect (default 11411).</param>
/// <param name="allowSequenceRestart">Whether to allow sequence ID restarts upon connection loss/reacquire.</param>
/// <remarks>In this case the start is a special behavior that is `DateTime.UtcNow` _at the sending `RemoteExporter`_.</remarks>
public RemoteImporter(Pipeline pipeline, string host, int port = RemoteExporter.DefaultPort, bool allowSequenceRestart = true)
: this(name => PsiStore.Open(pipeline, name, null), new TimeInterval(DateTime.MinValue, DateTime.MaxValue), true, host, port, $"RemoteImporter_{Guid.NewGuid().ToString()}", null, allowSequenceRestart)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="RemoteImporter"/> class.
/// </summary>
/// <param name="pipeline">Pipeline to which to attach.</param>
/// <param name="replay">Time interval to be replayed from remote source.</param>
/// <param name="host">Remote host name.</param>
/// <param name="port">TCP port on which to connect (default 11411).</param>
/// <param name="allowSequenceRestart">Whether to allow sequence ID restarts upon connection loss/reacquire.</param>
public RemoteImporter(Pipeline pipeline, TimeInterval replay, string host, int port = RemoteExporter.DefaultPort, bool allowSequenceRestart = true)
: this(name => PsiStore.Open(pipeline, name, null), replay, false, host, port, $"RemoteImporter_{Guid.NewGuid().ToString()}", null, allowSequenceRestart)
{
}
public void CrossFrameworkDeserialize()
{
if (!File.Exists(OtherTestExe))
{
Assert.Inconclusive($"Unable to locate {OtherTestExe} to generate the test store for {nameof(this.CrossFrameworkDeserializeMembers)}.");
}
this.ExecuteTest(OtherTestExe, $"!{nameof(this.CrossFrameworkSerialize)}");
int intValue = 0;
byte byteValue = 0;
bool boolValue = false;
short shortValue = 0;
long longValue = 0;
char charValue = '\0';
string stringValue = null;
double doubleValue = 0;
float floatValue = 0;
float[] floatArray = null;
List <string> stringList = null;
ArraySegment <string> stringArraySegment = default;
Queue <TimeSpan> queue = null;
EqualityComparer <int> intComparer = null;
Tuple <long, string> tuple = null;
(DateTime, Stack <int>)valueTuple = default;
Array intArray = null;
ICollection stringArray = null;
IEqualityComparer enumComparer = null;
using (var p = Pipeline.Create())
{
var store = PsiStore.Open(p, "Store1", this.testPath);
store.Serializers.Register <System.Drawing.Point>();
store.OpenStream <int>("int").Do(x => intValue = x);
store.OpenStream <byte>("byte").Do(x => byteValue = x);
store.OpenStream <bool>("bool").Do(x => boolValue = x);
store.OpenStream <short>("short").Do(x => shortValue = x);
store.OpenStream <long>("long").Do(x => longValue = x);
store.OpenStream <char>("char").Do(x => charValue = x);
store.OpenStream <string>("string").Do(x => stringValue = x);
store.OpenStream <double>("double").Do(x => doubleValue = x);
store.OpenStream <float>("float").Do(x => floatValue = x);
store.OpenStream <float[]>("floatArray").Do(x => floatArray = x.DeepClone());
store.OpenStream <List <string> >("stringList").Do(x => stringList = x.DeepClone());
store.OpenStream <ArraySegment <string> >("stringArraySegment").Do(x => stringArraySegment = x.DeepClone());
store.OpenStream <Queue <TimeSpan> >("queue").Do(x => queue = x.DeepClone());
store.OpenStream <EqualityComparer <int> >("intComparer").Do(x => intComparer = x.DeepClone());
store.OpenStream <Tuple <long, string> >("tuple").Do(x => tuple = x.DeepClone());
store.OpenStream <(DateTime, Stack <int>)>("dateStackTuple").Do(x => valueTuple = x.DeepClone());
store.OpenStream <Array>("intArray").Do(x => intArray = x.DeepClone());
store.OpenStream <ICollection>("stringArray").Do(x => stringArray = x.DeepClone());
store.OpenStream <IEqualityComparer>("enumComparer").Do(x => enumComparer = x.DeepClone());
p.Run();
}
Assert.AreEqual(0x7777AAA, intValue);
Assert.AreEqual(0xBB, byteValue);
Assert.AreEqual(true, boolValue);
Assert.AreEqual(0x7CDD, shortValue);
Assert.AreEqual(0x77777777EEEEEEEE, longValue);
Assert.AreEqual('G', charValue);
Assert.AreEqual("This is a test.", stringValue);
Assert.AreEqual(Math.PI, doubleValue);
Assert.AreEqual(-1.234f, floatValue);
CollectionAssert.AreEqual(new[] { 0.1f, 2.3f }, floatArray);
CollectionAssert.AreEqual(new[] { "one", "two" }, stringList);
CollectionAssert.AreEqual(new[] { "bbb", "ccc" }, stringArraySegment.ToArray());
CollectionAssert.AreEqual(new[] { TimeSpan.Zero, TimeSpan.FromSeconds(1) }, queue);
Assert.IsTrue(intComparer.Equals(1952, 1952));
Assert.AreEqual(0x77777777EEEEEEEE, tuple.Item1);
Assert.AreEqual("This is a tuple.", tuple.Item2);
Assert.AreEqual(new DateTime(2020, 1, 2), valueTuple.Item1);
CollectionAssert.AreEqual(new[] { 782, 33 }, valueTuple.Item2);
CollectionAssert.AreEqual(new[] { 0, 3 }, intArray);
CollectionAssert.AreEqual(new[] { "three", "four" }, stringArray);
Assert.IsTrue(enumComparer.Equals(DayOfWeek.Friday, DayOfWeek.Friday));
}
/// <summary>
/// Initializes a new instance of the <see cref="RemoteExporter"/> class.
/// </summary>
/// <param name="pipeline">Pipeline to which to attach.</param>
/// <param name="port">TCP port on which to listen (default 11411).</param>
/// <param name="transport">Transport kind to use.</param>
/// <param name="maxBytesPerSecond">Maximum bytes/sec quota (default infinite).</param>
/// <param name="bytesPerSecondSmoothingWindowSeconds">Smoothing window over which to compute bytes/sec (default 5 sec.).</param>
public RemoteExporter(Pipeline pipeline, int port = DefaultPort, TransportKind transport = DefaultTransport, long maxBytesPerSecond = long.MaxValue, double bytesPerSecondSmoothingWindowSeconds = 5.0)
: this(PsiStore.Create(pipeline, $"RemoteExporter_{Guid.NewGuid().ToString()}", null, true), port, transport, maxBytesPerSecond, bytesPerSecondSmoothingWindowSeconds)
{
}
/// <summary>
/// Execute task against each message in a stream.
/// </summary>
/// <param name="stream">Stream name.</param>
/// <param name="store">Store name.</param>
/// <param name="path">Store path.</param>
/// <param name="name">Task name.</param>
/// <param name="assemblies">Optional assemblies containing task.</param>
/// <param name="args">Additional configuration arguments.</param>
/// <returns>Success flag.</returns>
internal static int ExecuteTask(string stream, string store, string path, string name, IEnumerable <string> assemblies, IEnumerable <string> args)
{
Console.WriteLine($"Execute Task (stream={stream}, store={store}, path={path}, name={name}, assemblies={assemblies}, args={args})");
// find task
var tasks = LoadTasks(assemblies).Where(t => t.Attribute.Name == name).ToArray();
if (tasks.Length == 0)
{
throw new Exception($"Could not find task named '{name}'.");
}
else if (tasks.Length > 1)
{
throw new Exception($"Ambiguous task name ({tasks.Count()} tasks found named '{name}').");
}
var task = tasks[0];
// process task
using (var pipeline = Pipeline.Create())
{
var importer = store != null?PsiStore.Open(pipeline, store, Path.GetFullPath(path)) : null;
// prepare parameters
var streamMode = stream?.Length > 0;
var messageIndex = -1;
var envelopeIndex = -1;
var argList = args.ToArray();
var argIndex = 0;
var parameterInfo = task.Method.GetParameters();
var parameters = new object[parameterInfo.Length];
for (var i = 0; i < parameterInfo.Length; i++)
{
var p = parameterInfo[i];
if (p.ParameterType.IsAssignableFrom(typeof(Importer)))
{
if (importer == null)
{
throw new ArgumentException("Error: Task requires a store, but no store argument supplied (-s).");
}
parameters[i] = importer;
}
else if (p.ParameterType.IsAssignableFrom(typeof(Pipeline)))
{
parameters[i] = pipeline;
}
else if (p.ParameterType.IsAssignableFrom(typeof(Envelope)))
{
envelopeIndex = i;
}
else if (streamMode && messageIndex == -1)
{
messageIndex = i; // assumed first arg
}
else
{
Action <string, Func <string, object> > processArgs = (friendlyName, parser) =>
{
if (argIndex < args.Count())
{
// take from command-line args
try
{
parameters[i] = parser(argList[argIndex++]);
}
catch (Exception ex)
{
throw new ArgumentException($"Error: Parameter '{p.Name}' ({i}) expected {friendlyName}.", ex);
}
}
else
{
// get value interactively
do
{
try
{
Console.Write($"{p.Name} ({friendlyName})? ");
parameters[i] = parser(Console.ReadLine());
}
catch
{
Console.WriteLine($"Error: Expected {friendlyName}.");
}
}while (parameters[i] == null);
}
};
if (p.ParameterType.IsAssignableFrom(typeof(double)))
{
processArgs("double", v => double.Parse(v));
}
else if (p.ParameterType.IsAssignableFrom(typeof(int)))
{
processArgs("integer", v => int.Parse(v));
}
else if (p.ParameterType.IsAssignableFrom(typeof(bool)))
{
processArgs("boolean", v => bool.Parse(v));
}
else if (p.ParameterType.IsAssignableFrom(typeof(DateTime)))
{
processArgs("datetime", v => DateTime.Parse(v));
}
else if (p.ParameterType.IsAssignableFrom(typeof(TimeSpan)))
{
processArgs("timespan", v => TimeSpan.Parse(v));
}
else if (p.ParameterType.IsAssignableFrom(typeof(string)))
{
processArgs("string", v => v);
}
else
{
throw new ArgumentException($"Unexpected parameter type ({p.ParameterType}).");
}
}
}
if (streamMode)
{
if (importer == null)
{
throw new ArgumentException("Error: Task requires a stream within a store, but no store argument supplied (-s).");
}
importer.OpenDynamicStream(stream).Do((m, e) =>
{
if (messageIndex != -1)
{
parameters[messageIndex] = m;
}
if (envelopeIndex != -1)
{
parameters[envelopeIndex] = e;
}
task.Method.Invoke(null, parameters);
});
}
else
{
task.Method.Invoke(null, parameters);
}
if (importer != null)
{
pipeline.ProgressReportInterval = TimeSpan.FromSeconds(1);
pipeline.RunAsync(null, new Progress <double>(p => Console.WriteLine($"Progress: {p * 100.0:F2}%")));
pipeline.WaitAll();
}
}
return(0);
}
public async Task TranscribeConversationsAsync(IEnumerable <string> voiceSignatureStringUsers)
{
uint samplesPerSecond = 16000;
byte bitsPerSample = 16;
byte channels = 8; // 7 + 1 channels
var config = SpeechConfig.FromSubscription(this.SubscriptionKey, this.Region);
config.SetProperty("ConversationTranscriptionInRoomAndOnline", "true");
var stopRecognition = new TaskCompletionSource <int>();
using (var audioInput = AudioInputStream.CreatePushStream(AudioStreamFormat.GetWaveFormatPCM(samplesPerSecond, bitsPerSample, channels)))
{
var meetingID = Guid.NewGuid().ToString();
using (var conversation = await Conversation.CreateConversationAsync(config, meetingID))
{
// create a conversation transcriber using audio stream input
using (this.conversationTranscriber = new ConversationTranscriber(AudioConfig.FromStreamInput(audioInput)))
{
conversationTranscriber.Transcribing += (s, e) =>
{
this.SetText($"TRANSCRIBING: Text={e.Result.Text} SpeakerId={e.Result.UserId}");
};
conversationTranscriber.Transcribed += (s, e) =>
{
if (e.Result.Reason == ResultReason.RecognizedSpeech)
{
this.SetText($"TRANSCRIBED: Text={e.Result.Text} SpeakerId={e.Result.UserId}");
}
else if (e.Result.Reason == ResultReason.NoMatch)
{
this.SetText($"NOMATCH: Speech could not be recognized.");
}
};
conversationTranscriber.Canceled += (s, e) =>
{
this.SetText($"CANCELED: Reason={e.Reason}");
if (e.Reason == CancellationReason.Error)
{
this.SetText($"CANCELED: ErrorCode={e.ErrorCode}");
this.SetText($"CANCELED: ErrorDetails={e.ErrorDetails}");
this.SetText($"CANCELED: Did you update the subscription info?");
stopRecognition.TrySetResult(0);
}
};
conversationTranscriber.SessionStarted += (s, e) =>
{
this.SetText($"\nSession started event. SessionId={e.SessionId}");
};
conversationTranscriber.SessionStopped += (s, e) =>
{
this.SetText($"\nSession stopped event. SessionId={e.SessionId}");
this.SetText("\nStop recognition.");
stopRecognition.TrySetResult(0);
};
// Add participants to the conversation.
int i = 1;
foreach (var voiceSignatureStringUser in voiceSignatureStringUsers)
{
var speaker = Participant.From($"User{i++}", "en-US", voiceSignatureStringUser);
await conversation.AddParticipantAsync(speaker);
}
// Join to the conversation and start transcribing
await conversationTranscriber.JoinConversationAsync(conversation);
await conversationTranscriber.StartTranscribingAsync().ConfigureAwait(false);
using (var p = Pipeline.Create())
{
var store = PsiStore.Create(p, "Transcribe", @"D:\Temp");
var capture = new AudioCapture(p, WaveFormat.CreatePcm((int)samplesPerSecond, bitsPerSample, channels)).Write("Audio", store);
capture.Do(audio => audioInput.Write(audio.Data));
p.RunAsync();
// waits for completion, then stop transcription
await stopRecognition.Task;
}
await conversationTranscriber.StopTranscribingAsync().ConfigureAwait(false);
}
}
}
}
/// <summary>
/// This is the main code for our Multimodal Speech Detection demo.
/// </summary>
private void PerformMultiModalSpeechDetection()
{
Console.WriteLine("Initializing Psi.");
bool detected = false;
// First create our \Psi pipeline
using (var pipeline = Pipeline.Create("MultiModalSpeechDetection"))
{
// Register an event handler to catch pipeline errors
pipeline.PipelineExceptionNotHandled += Pipeline_PipelineException;
// Register an event handler to be notified when the pipeline completes
pipeline.PipelineCompleted += Pipeline_PipelineCompleted;
// Next create our Kinect sensor. We will be using the color images, face tracking, and audio from the Kinect sensor
var kinectSensorConfig = new KinectSensorConfiguration
{
OutputColor = true,
OutputAudio = true,
OutputBodies = true, // In order to detect faces using Kinect you must also enable detection of bodies
};
var kinectSensor = new KinectSensor(pipeline, kinectSensorConfig);
var kinectFaceDetector = new Microsoft.Psi.Kinect.Face.KinectFaceDetector(pipeline, kinectSensor, Microsoft.Psi.Kinect.Face.KinectFaceDetectorConfiguration.Default);
// Create our Voice Activation Detector
var speechDetector = new SystemVoiceActivityDetector(pipeline);
var convertedAudio = kinectSensor.Audio.Resample(WaveFormat.Create16kHz1Channel16BitPcm());
convertedAudio.PipeTo(speechDetector);
// Use the Kinect's face track to determine if the mouth is opened
var mouthOpenAsFloat = kinectFaceDetector.Faces.Where(faces => faces.Count > 0).Select((List <Microsoft.Psi.Kinect.Face.KinectFace> list) =>
{
if (!detected)
{
detected = true;
Console.WriteLine("Found your face");
}
bool open = (list[0] != null) ? list[0].FaceProperties[Microsoft.Kinect.Face.FaceProperty.MouthOpen] == Microsoft.Kinect.DetectionResult.Yes : false;
return(open ? 1.0 : 0.0);
});
// Next take the "mouthOpen" value and create a hold on that value (so that we don't see 1,0,1,0,1 but instead would see 1,1,1,1,0.8,0.6,0.4)
var mouthOpen = mouthOpenAsFloat.Hold(0.1);
// Next join the results of the speechDetector with the mouthOpen generator and only select samples where
// we have detected speech and that the mouth was open.
var mouthAndSpeechDetector = speechDetector.Join(mouthOpen, hundredMs).Select((t, e) => t.Item1 && t.Item2);
// Convert our speech into text
var speechRecognition = convertedAudio.SpeechToText(mouthAndSpeechDetector);
speechRecognition.Do((s, t) =>
{
if (s.Item1.Length > 0)
{
Console.WriteLine("You said: " + s.Item1);
}
});
// Create a stream of landmarks (points) from the face detector
var facePoints = new List <Tuple <System.Windows.Point, string> >();
var landmarks = kinectFaceDetector.Faces.Where(faces => faces.Count > 0).Select((List <Microsoft.Psi.Kinect.Face.KinectFace> list) =>
{
facePoints.Clear();
System.Windows.Point pt1 = new System.Windows.Point(
list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.EyeLeft].X,
list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.EyeLeft].Y);
facePoints.Add(Tuple.Create(pt1, string.Empty));
System.Windows.Point pt2 = new System.Windows.Point(
list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.EyeRight].X,
list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.EyeRight].Y);
facePoints.Add(Tuple.Create(pt2, string.Empty));
System.Windows.Point pt3 = new System.Windows.Point(
list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.MouthCornerLeft].X,
list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.MouthCornerLeft].Y);
facePoints.Add(Tuple.Create(pt3, string.Empty));
System.Windows.Point pt4 = new System.Windows.Point(
list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.MouthCornerRight].X,
list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.MouthCornerRight].Y);
facePoints.Add(Tuple.Create(pt4, string.Empty));
System.Windows.Point pt5 = new System.Windows.Point(
list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.Nose].X,
list[0].FacePointsInColorSpace[Microsoft.Kinect.Face.FacePointType.Nose].Y);
facePoints.Add(Tuple.Create(pt5, string.Empty));
return(facePoints);
});
// ********************************************************************
// Finally create a Live Visualizer using PsiStudio.
// We must persist our streams to a store in order for Live Viz to work properly
// ********************************************************************
// Create store for the data. Live Visualizer can only read data from a store.
var pathToStore = Environment.GetFolderPath(Environment.SpecialFolder.MyVideos);
var store = PsiStore.Create(pipeline, ApplicationName, pathToStore);
mouthOpen.Select(v => v ? 1d : 0d).Write("MouthOpen", store);
speechDetector.Select(v => v ? 1d : 0d).Write("VAD", store);
mouthAndSpeechDetector.Write("Join(MouthOpen,VAD)", store);
kinectSensor.Audio.Write("Audio", store);
var images = kinectSensor.ColorImage.EncodeJpeg(90, DeliveryPolicy.LatestMessage).Out;
images.Write("Images", store, true, DeliveryPolicy.LatestMessage);
landmarks.Write("FaceLandmarks", store);
// Run the pipeline
pipeline.RunAsync();
Console.WriteLine("Press any key to finish recording");
Console.ReadKey();
}
}
public void CreateAndStartPipeline()
{
this.pipeline = Pipeline.Create("Teams Pipeline", enableDiagnostics: this.botSettings.EnablePsiDiagnostics);
this.frameSourceComponent = new FrameSourceComponent(this.pipeline, logger);
var mpegConfig = Mpeg4WriterConfiguration.Default;
mpegConfig.ContainsAudio = false;
mpegConfig.ImageWidth = (uint)this.botSettings.Resize.Width;
mpegConfig.ImageHeight = (uint)this.botSettings.Resize.Height;
mpegConfig.PixelFormat = PixelFormat.BGR_24bpp;
var basePath = this.botSettings.RecordingFilePath;
var endpointUrl = this.botSettings.ModelEndpointUrl;
var resized = frameSourceComponent
.Video
.Select(v => v.First().Value) // discarding participant id - this means that no information of participant is carried forward
.Resize(this.botSettings.Resize.Width, this.botSettings.Resize.Height)
.Name("Resized Frames");
// input is stream of frames
var fileNames = resized
.WriteMP4InBatches(TimeSpan.FromSeconds(this.botSettings.VideoSegmentationIntervalInSeconds), basePath, mpegConfig)
.Name("FileNames");
var labelStream = fileNames
.CallModel(endpointUrl, basePath, logger).Name("Model Result")
.Do(l => this.logger.Info($"file: {l.filename} label: {l.label}"));
// output is stream of labels
labelStream.Item2()
.PerformTextToSpeech(this.ttsSettings, this.logger).Name("Text To Speech")
.Do(bytes => this.sendAudioToBot(CreateAudioMediaBuffers(DateTime.UtcNow.Ticks, bytes))).Name("Send Audio To Bot");
Generators
.Repeat(pipeline, true, TimeSpan.FromSeconds(1.0 / 15)).Name("15fps generation event")
.Pair(labelStream, DeliveryPolicy.LatestMessage, DeliveryPolicy.LatestMessage)
.Do(f =>
{
try
{
var text = f.Item3;
using (var sharedImage = ProduceScreenShare(text))
{
var image = sharedImage.Resource;
var nv12 = BGRAtoNV12(image.ImageData, image.Width, image.Height);
this.sendScreenShareToBot(nv12);
}
}
catch (Exception ex)
{
this.logger.Error(ex, "Error while screen sharing");
}
}).Name("Screen Share to bot");
if (this.botSettings.EnablePsiStore)
{
var store = PsiStore.Create(pipeline, "Bot", this.botSettings.PsiStorePath);
//resized.Write("video", store);
labelStream.Write("label", store);
if (this.botSettings.EnablePsiDiagnostics)
{
pipeline.Diagnostics.Write("Diagnostics", store);
}
}
this.pipeline.PipelineExceptionNotHandled += (_, ex) =>
{
this.logger.Error(ex.Exception, $"PSI PIPELINE ERROR: {ex.Exception.Message}");
};
pipeline.RunAsync();
//Task.Run(async () => {
// await Task.Delay(30000);
// this.logger.Warn("STOPPPPPPPPPIIIIIIIIINNNNNNNNGGGGGGGGGGGGGGGGG");
// pipeline.Dispose();
//});
}
