/// <summary>
/// Create audio buffers from a stream and populate the given list starting at the reference time tick.
/// </summary>
/// <param name="stream">The stream to read audio from</param>
/// <param name="audioBuffers">The list of audio buffers to be populated</param>
/// <param name="referenceTimeTick">The reference starting time tick</param>
private void CreateAudioBuffers(Stream stream, List <AudioMediaBuffer> audioBuffers, long referenceTimeTick)
{
// a 20ms buffer is 640 bytes at PCM16k
int bufferSize = 640;
byte[] bytesToRead = new byte[bufferSize];
// reset the input stream to initial position
stream.Position = 0;
// create 20ms buffers from the input stream
while (stream.Read(bytesToRead, 0, bytesToRead.Length) >= bufferSize)
{
IntPtr unmanagedBuffer = Marshal.AllocHGlobal(bufferSize);
Marshal.Copy(bytesToRead, 0, unmanagedBuffer, bufferSize);
// move the reference time by 20ms (there are 10.000 ticks in a milliseconds)
referenceTimeTick += 20 * 10000;
// create the audio buffer and add it to the list
var audioBuffer = new AudioSendBuffer(unmanagedBuffer, bufferSize, AudioFormat.Pcm16K,
referenceTimeTick);
audioBuffers.Add(audioBuffer);
}
Log.Info(
new CallerInfo(),
LogContext.Media,
"created {0} AduioMediaBuffers frames", audioBuffers.Count);
}
public static List <AudioMediaBuffer> CreateAudioMediaBuffers(long currentTick)
{
var audioMediaBuffers = new List <AudioMediaBuffer>();
var referenceTime = currentTick;
using (FileStream fs = File.Open(Service.Instance.Configuration.AudioFileLocation, FileMode.Open))
{
byte[] bytesToRead = new byte[640];
fs.Seek(44, SeekOrigin.Begin);
while (fs.Read(bytesToRead, 0, bytesToRead.Length) >= 640) //20ms
{
IntPtr unmanagedBuffer = Marshal.AllocHGlobal(640);
Marshal.Copy(bytesToRead, 0, unmanagedBuffer, 640);
referenceTime += 20 * 10000;
var audioBuffer = new AudioSendBuffer(unmanagedBuffer, 640, AudioFormat.Pcm16K,
referenceTime);
audioMediaBuffers.Add(audioBuffer);
}
}
Log.Info(
new CallerInfo(),
LogContext.Media,
"created {0} AudioMediaBuffers", audioMediaBuffers.Count);
return(audioMediaBuffers);
}
/// <summary>
/// Create audio buffers from a stream and populate the given list starting at the reference time tick.
/// </summary>
/// <param name="stream">The stream to read audio from</param>
/// <param name="audioBuffers">The list of audio buffers to be populated</param>
/// <param name="referenceTimeTick">The reference starting time tick</param>
private long populateAudioBuffersFromStream(Stream stream, List <AudioMediaBuffer> audioBuffers, long referenceTimeTick)
{
// a 20ms buffer is 640 bytes
int bufferSize = 640;
byte[] bytesToRead = new byte[bufferSize];
// reset the input stream to initial position
stream.Position = 0;
// create 20ms buffers from the input stream
while (stream.Read(bytesToRead, 0, bytesToRead.Length) >= bufferSize)
{
IntPtr unmanagedBuffer = Marshal.AllocHGlobal(bufferSize);
Marshal.Copy(bytesToRead, 0, unmanagedBuffer, bufferSize);
// move the reference time by 20ms (there are 10.000 ticks in a milliseconds)
referenceTimeTick += 20 * 10000;
// create the audio buffer and add it to the list
var audioBuffer = new AudioSendBuffer(unmanagedBuffer, bufferSize, AudioFormat.Pcm16K, referenceTimeTick);
audioBuffers.Add(audioBuffer);
}
// return the reference time tick of the last buffer so we can queue new data if needed
return(referenceTimeTick);
}
/// <summary>
/// Helper function to create the audio buffers from file.
/// Please make sure the audio file provided is PCM16Khz and the fileSizeInSec is the correct length.
/// </summary>
/// <param name="currentTick">The current clock tick.</param>
/// <param name="replayed">Whether it's replayed.</param>
/// <param name="logger">Graph logger.</param>
/// <returns>The newly created list of <see cref="AudioMediaBuffer"/>.</returns>
public static List <AudioMediaBuffer> CreateAudioMediaBuffers(long currentTick, bool replayed, IGraphLogger logger)
{
var audioMediaBuffers = new List <AudioMediaBuffer>();
var referenceTime = currentTick;
// packet size of 20 ms
var numberOfTicksInOneAudioBuffers = 20 * 10000;
if (replayed)
{
referenceTime += numberOfTicksInOneAudioBuffers;
}
using (FileStream fs = File.Open(Service.Instance.Configuration.AudioFileLocation, FileMode.Open))
{
byte[] bytesToRead = new byte[640];
// skipping the wav headers
fs.Seek(44, SeekOrigin.Begin);
while (fs.Read(bytesToRead, 0, bytesToRead.Length) >= 640)
{
// here we want to create buffers of 20MS with PCM 16Khz
IntPtr unmanagedBuffer = Marshal.AllocHGlobal(640);
Marshal.Copy(bytesToRead, 0, unmanagedBuffer, 640);
var audioBuffer = new AudioSendBuffer(unmanagedBuffer, 640, AudioFormat.Pcm16K, referenceTime);
audioMediaBuffers.Add(audioBuffer);
referenceTime += numberOfTicksInOneAudioBuffers;
}
}
logger.Info($"created {audioMediaBuffers.Count} AudioMediaBuffers");
return(audioMediaBuffers);
}
public void OnClientAudioReceived(AudioFrame frame)
{
this.audioFrameCounter += 1;
int inputBufferLength = (int)(frame.sampleRate / 100 * frame.bitsPerSample / 8 * frame.channelCount);
byte[] inputBuffer = new byte[inputBufferLength];
Marshal.Copy(frame.audioData, inputBuffer, 0, inputBufferLength);
byte[] pcm16Bytes = AudioConverter.ResampleAudio(inputBuffer, (int)frame.sampleRate,
(int)frame.bitsPerSample, (int)frame.channelCount, Config.AudioSettings.OUTGOING_SAMPLE_RATE);
if (this.audioFrameCounter % this.maxAudioFramesToSend == 0)
{
try
{
byte[] stackedFrames = AudioConverter.MergeFrames(this.savedFrame, pcm16Bytes);
IntPtr pcm16Pointer = Marshal.AllocHGlobal(stackedFrames.Length);
Marshal.Copy(stackedFrames, 0, pcm16Pointer, stackedFrames.Length);
var audioSendBuffer = new AudioSendBuffer(pcm16Pointer, (long)stackedFrames.Length, AudioFormat.Pcm16K);
this.Call.GetLocalMediaSession().AudioSocket.Send(audioSendBuffer);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
else
{
this.savedFrame = pcm16Bytes;
}
}
/// <summary>
/// Callback from the media platform when raw audio received. This method sends the raw
/// audio to the transcriber. The audio is also loopbacked to the user.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void OnAudioMediaReceived(object sender, AudioMediaReceivedEventArgs e)
{
if (!_sendAudio)
{
e.Buffer.Dispose();
return;
}
CorrelationId.SetCurrentId(_correlationId);
Log.Verbose(
new CallerInfo(),
LogContext.Media,
"[{0}] [AudioMediaReceivedEventArgs(Data=<{1}>, Length={2}, Timestamp={3}, AudioFormat={4})]",
this.Id,
e.Buffer.Data.ToString(),
e.Buffer.Length,
e.Buffer.Timestamp,
e.Buffer.AudioFormat);
try
{
var audioSendBuffer = new AudioSendBuffer(e.Buffer, AudioFormat.Pcm16K, (UInt64)DateTime.Now.Ticks);
_audioSocket.Send(audioSendBuffer);
byte[] buffer = new byte[e.Buffer.Length];
Marshal.Copy(e.Buffer.Data, buffer, 0, (int)e.Buffer.Length);
//If the recognize had completed with error/timeout, the underlying stream might have been swapped out on us and disposed.
//so ignore the objectDisposedException
try
{
_recognitionStream.Write(buffer, 0, buffer.Length);
}
catch (ObjectDisposedException)
{
Log.Info(new CallerInfo(), LogContext.Media, $"[{this.Id}]: Write on recognitionStream threw ObjectDisposed");
}
}
catch (Exception ex)
{
Log.Error(new CallerInfo(), LogContext.Media, $"[{this.Id}]: Caught exception when attempting to send audio buffer {ex.ToString()}");
}
finally
{
e.Buffer.Dispose();
}
}
public List <AudioMediaBuffer> CreateAudioMediaBuffers(long currentTick, Byte[] audio)
{
var stream = new MemoryStream(audio);
var audioMediaBuffers = new List <AudioMediaBuffer>();
var referenceTime = currentTick;
using (stream)
{
byte[] bytesToRead = new byte[640];
stream.Seek(44, SeekOrigin.Begin);
while (stream.Read(bytesToRead, 0, bytesToRead.Length) >= 640) //20ms
{
IntPtr unmanagedBuffer = Marshal.AllocHGlobal(640);
Marshal.Copy(bytesToRead, 0, unmanagedBuffer, 640);
referenceTime += 20 * 10000;
var audioBuffer = new AudioSendBuffer(unmanagedBuffer, 640, AudioFormat.Pcm16K,
referenceTime);
audioMediaBuffers.Add(audioBuffer);
}
}
return(audioMediaBuffers);
}
public List <AudioMediaBuffer> GetAudioMediaBuffers(long currentTick)
{
Stopwatch watch = new Stopwatch();
watch.Start();
// 1. Downlaod _nbSecondToLoad seconds of audio content from the storage account
long bufferSize = 16000 * 2 * _nbSecondToLoad; // Pcm16K is 16000 samples per seconds, each sample is 2 bytes
byte[] bytesToRead = new byte[bufferSize];
var nbByteRead = _audioBlob.DownloadRangeToByteArray(bytesToRead, 0, _audioOffset, bytesToRead.Length, null, null);
//2. Extract each audio sample in a AudioMediaBuffer object
List <AudioMediaBuffer> audioMediaBuffers = new List <AudioMediaBuffer>();
int audioBufferSize = (int)(16000 * 2 * 0.02); // the Real-time media platform expects audio buffer duration of 20ms
long referenceTime = currentTick;
for (int index = 0; index < nbByteRead; index += audioBufferSize)
{
IntPtr unmanagedBuffer = Marshal.AllocHGlobal(audioBufferSize);
Marshal.Copy(bytesToRead, index, unmanagedBuffer, audioBufferSize);
// 10000 ticks in a ms
referenceTime += 20 * 10000;
var audioBuffer = new AudioSendBuffer(unmanagedBuffer, audioBufferSize, _audioFormat, referenceTime);
audioMediaBuffers.Add(audioBuffer);
_audioOffset += audioBufferSize;
}
Log.Info(new CallerInfo(), LogContext.FrontEnd, $"Loading {_nbSecondToLoad}s audio took {watch.ElapsedMilliseconds}ms ({16000 * 2 * _nbSecondToLoad} bytes)");
watch.Stop();
return(audioMediaBuffers);
}