/// <summary>
/// If not more than three notes are playing at once, the player plays a sound
/// corresponding to the color hue. The rythm is determined based on
/// color change and the volume is determined based on saturation.
/// </summary>
/// <param name="previous_color"></param>
/// <param name="current_color"></param>
public void PlayColor(UIColor previous_color, UIColor current_color)
{
if (IsPlayingMoreThanN(12))
{
// more than N players are playing so we do nothing since
// it would overcrowd the audio stream
return;
}
double rythm = ColorProcessor.GetRythm(previous_color, current_color);
// hue ranges from 0-360. We are going to use this value for mapping
// to sound.
current_color.GetHSBA(out nfloat hue, out _, out _, out _);
int index = HueToIndex(hue);
// we base volume on color saturation
number_to_sound[index].Volume = 1.0f;
// we base playback speed on color change defined in GetRythm
number_to_sound[index].Rate = (float)rythm;
// we finally play the sound
number_to_sound[index].Play();
}
/// <summary>
/// Callback method for when a new image is recieved from the video.
/// </summary>
public override void DidOutputSampleBuffer(AVCaptureOutput captureOutput, CMSampleBuffer sampleBuffer, AVCaptureConnection connection)
{
var imageBuffer = sampleBuffer.GetImageBuffer();
var ciImage = new CIImage(imageBuffer);
var cgImage = this.context.CreateCGImage(ciImage, ciImage.Extent);
var current_color = ColorProcessor.AverageColor(cgImage);
if (musicProcessor == null)
{
Console.WriteLine("Music processor is null. Waiting for init.");
}
else if (!recording)
{
musicProcessor.PlayColor(current_color, previous_color);
}
previous_color = current_color;
if (!recording)
{
// we intentionally do not await these methods since it would
// prohibit the rest of the app from working.
// these methods are executed depending on the durationn of
// the API call to Azure.
// we handle the busy part here since if we handled it in PlayObjects
// it would overcrowd the threads for async calls.
if (generalObjectRecognition && !objectsProcessor.busy)
{
objectsProcessor.PlayObjects(cgImage);
}
if (customObjectRecognition && !customObjectsProcessor.busy)
{
customObjectsProcessor.PlayObjects(cgImage);
}
}
BeginInvokeOnMainThread(async() =>
{
// we need to change the background color on the main thread
// this thread is the UI thred
background.BackgroundColor = current_color;
});
// we dispose the sample buffer since we are not able to process
// all the images and we would overcrowd
sampleBuffer.Dispose();
}