/// <summary>
/// This event handler is called when the robot/user sends a start message
/// The parameters can be set in the Skill Runner (or as json) and used in the skill if desired
/// </summary>
/// <param name="parameters"></param>
public async void OnStart(object sender, IDictionary <string, object> parameters)
{
//TODO Put your code here and update the summary above
int task_id = 0;
_timeToTellJokes = true;
try
{
//Get the audio and image lists for use in the skill
_audioList = (await _misty.GetAudioListAsync())?.Data;
_imageList = (await _misty.GetImageListAsync())?.Data;
_misty.Wait(2000);
task_id++;
_misty.PlayAudio("Misty_Hi.wav", 80, null);
_misty.Wait(2000);
_misty.PlayAudio("Misty_I_am_Annie.wav", 80, null);
_misty.Wait(4000);
_misty.ChangeLED(255, 255, 255, null);
_misty.DisplayImage("e_DefaultContent.jpg", 1, null);
_misty.MoveHead(10, 0, 0, 60, AngularUnit.Degrees, null);
task_id++;
_misty.RegisterAudioPlayCompleteEvent(AudioPlayCallback, 0, true, null, null);
// Temporarily disable this voice recognition of the keyphrase wake up words due to SDK software issue.
// It will be used the the fix is done.
// The skill will be initiated by the skill sequencer.
// _misty.StartKeyPhraseRecognition(null);
// _misty.RegisterKeyPhraseRecognizedEvent(10, false, "KeyPhrase", null);
// _misty.KeyPhraseRecognizedEventReceived += ProcessKeyPhraseEvent;
_misty.StartFaceRecognition(null);
RegisterEvents();
task_id++;
_heartbeatTimer = new Timer(HeartbeatCallback, null, 5000, 3000);
_moveHeadTimer = new Timer(MoveHeadCallback, null, 5000, 7000);
_moveArmsTimer = new Timer(MoveArmCallback, null, 5000, 4000);
_ledTimer = new Timer(ChangeLEDCallback, null, 1000, 1000);
}
catch (Exception ex)
{
if (task_id == 0)
{
_misty.SkillLogger.LogVerbose($"TellingJokeSkill : Failed to Load audio and image files");
}
else if (task_id == 1)
{
_misty.SkillLogger.LogVerbose($"TellingJokeSkill : Failed to play audio and display image files");
}
else if (task_id == 2)
{
_misty.SkillLogger.LogVerbose($"TellingJokeSkill : Failed to register events");
}
else
{
_misty.SkillLogger.LogVerbose($"TellingJokeSkill : Failed to setup timers");
}
_misty.SkillLogger.Log($"TellingJokeSkill : OnStart: => Exception", ex);
}
}
/// <summary>
/// Get the assets and startup some timers to do random movements and other things...
/// </summary>
/// <param name="parameters"></param>
public async void OnStart(object sender, IDictionary<string, object> parameters)
{
try
{
//Get the audio and image lists for use in the skill
_audioList = (await _misty.GetAudioListAsync())?.Data;
_imageList = (await _misty.GetImageListAsync())?.Data;
_misty.Wait(2000);
_misty.PlayAudio("s_Awe.wav", 100, null);
_misty.ChangeLED(255, 255, 255, null);
_misty.DisplayImage("e_ContentRight.jpg", 1, null);
RegisterEvents();
//_heartbeatTimer = new Timer(HeartbeatCallback, null, 3000, 3000);
_moveHeadTimer = new Timer(MoveHeadCallback, null, 10000, 10000);
_moveArmsTimer = new Timer(MoveArmCallback, null, 10000, 10000);
_ledTimer = new Timer(ChangeLEDCallback, null, 2000, 2000);
}
catch (Exception ex)
{
_misty.SkillLogger.Log($"ForceDriving : OnStart: => Exception", ex);
}
}
/// <summary>
/// Called when the robot wants to start this skill
/// </summary>
/// <param name="parameters"></param>
public void OnStart(object sender, IDictionary <string, object> parameters)
{
try
{
//These calls assume system assets on the robot at the time of writing.
//Update as needed for new or different assets or as an exercise to allow user to pass in asset names :)
_misty.PlayAudio("s_Acceptance.wav", 100, null);
_misty.MoveHead(45, 25, 0, 50, AngularUnit.Degrees, null);
_misty.MoveArms(0, 45, 25, 60, null, AngularUnit.Degrees, null);
_misty.ChangeLED(0, 255, 0, null);
_misty.DisplayImage("e_Disoriented.jpg", 1, null);
//Pause for 4 seconds, if the cancellation token is set during this time, exit the pause and the method
if (!_misty.Wait(4000))
{
return;
}
_misty.PlayAudio("s_Awe2.wav", 100, null);
_misty.MoveHead(75, 15, 30, 50, AngularUnit.Degrees, null);
_misty.MoveArms(-45, 0, 60, 100, null, AngularUnit.Degrees, null);
_misty.ChangeLED(0, 255, 255, null);
_misty.DisplayImage("e_ContentRight.jpg", 1, null);
//Pause for 3.5 seconds, if the cancellation token is set during this time, exit the pause and the method
if (!_misty.Wait(3500))
{
return;
}
_misty.PlayAudio("s_Joy.wav", 100, null);
_misty.MoveHead(75, 25, 10, 50, AngularUnit.Degrees, null);
_misty.MoveArms(-45, 45, 60, 100, null, AngularUnit.Degrees, null);
_misty.ChangeLED(255, 255, 255, null);
_misty.DisplayImage("e_ContentLeft.jpg", 1, null);
//Pause for 2.5 seconds, if the cancellation token is set during this time, exit the pause and the method
if (!_misty.Wait(2500))
{
return;
}
_misty.PlayAudio("s_PhraseHello.wav", 100, null);
_misty.MoveHead(-10, 0, -10, 50, AngularUnit.Degrees, null);
_misty.MoveArms(0, -45, 60, 100, null, AngularUnit.Degrees, null);
_misty.ChangeLED(0, 0, 255, null);
_misty.DisplayImage("e_Joy.jpg", 1, null);
//Tell the robot the skill has completed early
_misty.SkillCompleted();
}
catch (Exception ex)
{
_misty.SkillLogger.Log($"HelloWorldSkill : OnStart: => Exception", ex);
}
}
/// <summary>
/// Converts the text value to an audio file and then plays the audio file.
/// </summary>
/// <param name="textToSpeak">stores the text to be converted to audio</param>
public async Task <bool> SpeakTheTextAsync(string textToSpeak)
{
_misty.ChangeLED(0, 255, 255, null); // light blue
IsBusy = true; // Set busy flag so others will wait until not busy speaking the text.
try
{
_misty.SkillLogger.LogVerbose($"MistySpeechApi : IN speakTheTextAsync -- textToSpeak = XX{textToSpeak}XX");
// Parameters to send with request to Google TTS API.
// For more information see the Google TTS API developer docs:
// https://cloud.google.com/text-to-speech/docs/reference/rest/v1beta1/text/synthesize
var arguments = @"{
'input': {
'text': '" + textToSpeak +
@"'},
'voice': {
'languageCode': 'en-US',
'ssmlGender': 'FEMALE'
},
'audioConfig': {
'audioEncoding': 'LINEAR16',
'effectsProfileId': [
'handset-class-device'
],
'pitch': 0,
'speakingRate': 0.91
}
}";
_misty.SkillLogger.LogVerbose("SendExternalRequestAsync - Sending TextToSpeak to Google TTS Api");
ISendExternalRequestResponse sdata = await _misty.SendExternalRequestAsync("POST", MistyApiInfo.GoogleTextToSpeechUrl, "Bearer", GoogleAuthToken, arguments, false, false, null, "application/json");
_misty.SkillLogger.LogVerbose("SendExternalRequestAsync - BACK FROM Google TTS Api");
string jsonStrResponse = sdata.Data.Data.ToString();
Newtonsoft.Json.Linq.JObject jsonResp = Newtonsoft.Json.Linq.JObject.Parse(jsonStrResponse);
byte[] audio_data = Convert.FromBase64String(jsonResp["audioContent"].ToString());
string audio_filename = "tts_response.wav";
// Save the base64 audio file.
_misty.SaveAudio(audio_filename, audio_data, true, true, null);
_misty.Wait(5000);
_misty.ChangeLED(0, 255, 0, null); // Green
return(true);
}
catch (Exception ex)
{
_misty.SkillLogger.Log($"MistySpeechApi: IN SpeechResponseCallback: => Exception", ex);
return(true);
}
finally
{
IsBusy = false;
}
}
/// <summary>
/// Called when the robot wants to start this skill
/// </summary>
/// <param name="parameters"></param>
public void OnStart(object sender, IDictionary <string, object> parameters)
{
_misty.SkillLogger.LogVerbose($"MostlyHarmlessSkill : OnStart called => Start a loop to change the LED every 250 ms");
Random randomGenerator = new Random();
//Pause for 100 ms, if the cancellation token is set during this time, exit the pause and the method
while (_misty.Wait(100))
{
_misty.ChangeLED((uint)randomGenerator.Next(0, 256), (uint)randomGenerator.Next(0, 256), (uint)randomGenerator.Next(0, 256), null);
}
}
public async void OnStart(object sender, IDictionary <string, object> parameters)
{
//callback example
_misty.ChangeLED(255, 36, 0, OnResponse);
_misty.Wait(2000);
//second callback example
_misty.ChangeLED(0, 36, 255, OnResponse);
_misty.Wait(1500);
//async example
await _misty.ChangeLEDAsync(255, 140, 0);
//event registration
_misty.RegisterBumpSensorEvent(BumpCallback, 0, true, null, null, null);
}
/// <summary>
/// This event handler is called when the robot/user sends a start message
/// The parameters can be set in the Skill Runner (or as json) and used in the skill if desired
/// </summary>
/// <param name="parameters"></param>
public void OnStart(object sender, IDictionary <string, object> parameters)
{
_misty.ChangeLED(255, 255, 0, null);
OnQueryResultReceived += ProcessQueryResult;
_client = new HttpClient();
_misty.RegisterBumpSensorEvent(onBumped, 20, true, null, "bumped", null);
_misty.RegisterCapTouchEvent(onTouched, 20, true, null, "toched", null);
try
{
_credential = getAuthToken()?.Result;
}
catch (Exception ex)
{
_misty.SkillLogger.Log("Unable to authenticate dialogflow client.", ex);
}
}
public void OnStart(object sender, IDictionary <string, object> parameters)
{
_misty.TransitionLED(255, 140, 0, 0, 0, 255, LEDTransition.Breathe, 1000, null);
_misty.Wait(3000);
_misty.ChangeLED(0, 0, 255, null);
ProcessParameters(parameters);
RegisterEvents();
//Start listening for heartbeat ticks
_driveHeartbeat.HeartbeatTick += HeartbeatCallback;
}
public void OnStart(object sender, IDictionary <string, object> parameters)
{
_misty.TransitionLED(255, 140, 0, 0, 0, 255, LEDTransition.Breathe, 1000, null);
_misty.Wait(3000);
_misty.ChangeLED(0, 0, 255, null);
//Register Bump Sensors with a callback
_misty.RegisterBumpSensorEvent(BumpCallback, 0, true, null, null, null);
//Register Cap Touch with a callback
_misty.RegisterCapTouchEvent(CapTouchCallback, 0, true, null, null, null);
}
public void OnStart(object sender, IDictionary <string, object> parameters)
{
_misty.TransitionLED(255, 140, 0, 0, 0, 255, LEDTransition.Breathe, 1000, null);
_misty.Wait(3000);
_misty.ChangeLED(0, 0, 255, null);
ProcessParameters(parameters);
//Randomly moves the head every X ms
_moveHeadTimer = new Timer(MoveHeadCallback, null, 0, _moveHeadPauseInMilliseconds);
//Randomly moves the arms every X ms
_moveArmsTimer = new Timer(MoveArmCallback, null, 0, _moveArmPauseInMilliseconds);
}
private void HandeDebugLocomotionMode()
{
if (_debugMode)
{
switch (_locomotionStatus)
{
case LocomotionStatus.DrivingBackward:
_misty.ChangeLED(255, 240, 0, null);
break;
case LocomotionStatus.DrivingForward:
_misty.ChangeLED(0, 255, 0, null);
break;
case LocomotionStatus.DrivingForwardLeft:
_misty.ChangeLED(0, 125, 255, null);
break;
case LocomotionStatus.DrivingForwardRight:
_misty.ChangeLED(0, 255, 127, null);
break;
case LocomotionStatus.Stopped:
_misty.ChangeLED(255, 0, 0, null);
break;
case LocomotionStatus.TurningLeft:
_misty.ChangeLED(255, 255, 0, null);
break;
case LocomotionStatus.TurningRight:
_misty.ChangeLED(255, 0, 255, null);
break;
default:
//Unknown
_misty.ChangeLED(255, 255, 255, null);
break;
}
}
}
public async void OnStart(object sender, IDictionary <string, object> parameters)
{
ProcessParameters(parameters);
await _misty.EnableAudioServiceAsync();
_misty.TransitionLED(255, 140, 0, 0, 0, 255, LEDTransition.Breathe, 500, null);
_misty.Wait(3000);
_misty.ChangeLED(0, 0, 255, null);
//Simple example of database use to track number of runs
await LogNumberOfRuns();
//Load assets if they do not exist on the robot or if reloadAssets sent in
await LoadAssets();
//Example of using a timer to play an uploaded sound every X ms
_playSoundTimer = new Timer(PlaySoundCallback, null, 0, 15000);
//Example of kicking off a side thread for the text display
_ = Task.Run(() => DisplayTextLoop());
//Example of setting a display setting and looping through simple display changes...
await _misty.SetImageDisplaySettingsAsync
(
"AssetFunLayer2",
new ImageSettings
{
Stretch = ImageStretch.Fill
}
);
//_misty.Wait(milliseconds) will wait (approx.) that amount of time
//if the skill is cancelled, it will return immediately with a false boolean response
while (_misty.Wait(5000))
{
await _misty.DisplayImageAsync("AssetFunSkillPumpkin1.jpg", "AssetFunLayer2", false);
_misty.Wait(5000);
await _misty.DisplayImageAsync("AssetFunSkillPumpkin2.jpg", "AssetFunLayer2", false);
}
}
/// <summary>
/// This event handler is called when the robot/user sends a start message
/// The parameters can be set in the Skill Runner (or as json) and used in the skill if desired
/// </summary>
/// <param name="parameters"></param>
public void OnStart(object sender, IDictionary <string, object> parameters)
{
//TODO Put your code here and update the summary above
_misty.ChangeLED(0, 36, 255, OnResponse);
_misty.Wait(1000);
_misty.ChangeLED(0, 0, 0, OnResponse);
_misty.Wait(1000);
_misty.ChangeLED(0, 36, 255, OnResponse);
_misty.Wait(1000);
_misty.ChangeLED(0, 0, 0, OnResponse);
_misty.Wait(1000);
_misty.ChangeLED(0, 36, 255, OnResponse);
_misty.Wait(1000);
_misty.ChangeLED(0, 0, 0, OnResponse);
_misty.Wait(1000);
}
/// <summary>
/// This event handler is called when the robot/user sends a start message
/// The parameters can be set in the Skill Runner (or as json) and used in the skill if desired
/// </summary>
/// <param name="parameters"></param>
public void OnStart(object sender, IDictionary <string, object> parameters)
{
//TODO Put your code here and update the summary above
_misty.ChangeLED(255, 0, 0, OnResponse);
clearAllMaps();
}
/// <summary>
/// Get the assets and startup some timers to do random movements and other things...
/// </summary>
/// <param name="parameters"></param>
public async void OnStart(object sender, IDictionary <string, object> parameters)
{
try
{
var deviceInfo = await _misty.GetDeviceInformationAsync();
if (!_misty.Wait(0))
{
return;
}
//Get the audio and image lists for use in the skill
_audioList = (await _misty.GetAudioListAsync())?.Data;
if (!_misty.Wait(0))
{
return;
}
_imageList = (await _misty.GetImageListAsync())?.Data;
if (!_misty.Wait(0))
{
return;
}
if (_audioList != null && _audioList.Count > 0)
{
AudioDetails randomAudio = _audioList[_randomGenerator.Next(0, _audioList.Count - 1)];
_misty.PlayAudio(randomAudio.Name, 100, null);
}
if (_imageList != null && _imageList.Count > 0)
{
ImageDetails randomImage = _imageList[_randomGenerator.Next(0, _imageList.Count - 1)];
_misty.DisplayImage(randomImage.Name, 1, null);
}
if (!_misty.Wait(2000))
{
return;
}
_misty.ChangeLED(255, 255, 255, null);
//Register a number of events
_misty.RegisterAudioPlayCompleteEvent(AudioPlayCallback, 0, true, null, null);
_misty.RegisterCapTouchEvent(CapTouchCallback, 0, true, null, null, null);
_misty.RegisterKeyPhraseRecognizedEvent(KeyPhraseRecognizedCallback, 250, true, null, null);
_misty.StartKeyPhraseRecognition(null);
_misty.StartFaceRecognition(null);
//Create an event with a specific name so we can unregister it when needed using that name
_misty.RegisterBumpSensorEvent(BumpCallback, 0, true, null, "MyBumpSensorName", null);
//Register face rec with keepAlive = false, it will need to be reregistered after triggering if the user wants it to run again
_misty.RegisterFaceRecognitionEvent(FaceRecCallback, 0, false, null, null, null);
//Play audio indicator that the event registration state has changed
if (_audioList != null && _audioList.Count > 0)
{
AudioDetails randomAudio = _audioList[_randomGenerator.Next(0, _audioList.Count - 1)];
_misty.PlayAudio(randomAudio.Name, 100, null);
await Task.Delay(1000);
_misty.PlayAudio(randomAudio.Name, 100, null);
}
if (!_misty.Wait(30000))
{
return;
}
//Unregister the bump sensor
_misty.UnregisterEvent("MyBumpSensorName", UnregisterCallback);
//Play audio indicator that the event registration state has changed
if (_audioList != null && _audioList.Count > 0)
{
AudioDetails randomAudio = _audioList[_randomGenerator.Next(0, _audioList.Count - 1)];
_misty.PlayAudio(randomAudio.Name, 100, null);
await Task.Delay(1000);
_misty.PlayAudio(randomAudio.Name, 100, null);
}
if (!_misty.Wait(20000))
{
return;
}
//Unregister ALL events
_misty.UnregisterAllEvents(UnregisterCallback);
//Play audio indicator that the event registration state has changed
if (_audioList != null && _audioList.Count > 0)
{
AudioDetails randomAudio = _audioList[_randomGenerator.Next(0, _audioList.Count - 1)];
_misty.PlayAudio(randomAudio.Name, 100, null);
await Task.Delay(1000);
_misty.PlayAudio(randomAudio.Name, 100, null);
}
if (!_misty.Wait(20000))
{
return;
}
//Play audio indicator that the event registration state has changed
if (_audioList != null && _audioList.Count > 0)
{
AudioDetails randomAudio = _audioList[_randomGenerator.Next(0, _audioList.Count - 1)];
_misty.PlayAudio(randomAudio.Name, 100, null);
await Task.Delay(1000);
_misty.PlayAudio(randomAudio.Name, 100, null);
}
//Re-register events
_misty.RegisterAudioPlayCompleteEvent(AudioPlayCallback, 0, true, null, null);
_misty.RegisterFaceRecognitionEvent(FaceRecCallback, 0, false, null, null, null);
_misty.RegisterCapTouchEvent(CapTouchCallback, 0, true, null, null, null);
_misty.RegisterKeyPhraseRecognizedEvent(KeyPhraseRecognizedCallback, 0, true, null, null);
//You can also register events without callbacks, it requires a user to subscribe to that event as follows...
//Note that this re-registers bump events to play a sound on release, not contact as they were previously handled
_misty.RegisterBumpSensorEvent(0, true, null, null, null);
_misty.BumpSensorEventReceived += ProcessBumpEvent;
//Will continue to process events until timeout of cancel
}
catch (Exception ex)
{
_misty.SkillLogger.Log($"InteractiveMistySkill : OnStart: => Exception", ex);
}
}
/// <summary>
/// Called when a user or the robot calls cancel on this skill
/// </summary>
public void OnCancel(object sender, IDictionary <string, object> parameters)
{
_misty.SkillLogger.LogVerbose($"MostlyHarmlessTooSkill : OnCancel called => Change LED to Red");
_misty.ChangeLED(255, 0, 0, null);
}
/// <summary>
/// This event handler is called when the robot/user sends a start message
/// The parameters can be set in the Skill Runner (or as json) and used in the skill if desired
/// </summary>
/// <param name="sender"></param>
/// <param name="parameters"></param>
public async void OnStart(object sender, IDictionary <string, object> parameters)
{
try
{
await InitializeSkill();
await ProcessStartupParameters(parameters);
await ProcessAdjustableParameters(parameters);
_misty.ChangeLED(255, 0, 0, null);
if (!_misty.Wait(0))
{
return;
}
//Spanish translations
List <BumpSensorValidation> backLeftBumpValidations = new List <BumpSensorValidation>();
backLeftBumpValidations.Add(new BumpSensorValidation(BumpSensorFilter.SensorName, ComparisonOperator.Equal, BumpSensorPosition.BackLeft));
backLeftBumpValidations.Add(new BumpSensorValidation(BumpSensorFilter.IsContacted, ComparisonOperator.Equal, true));
_misty.RegisterBumpSensorEvent(ProcessBackLeftBumpEvent, 250, true, backLeftBumpValidations, null, null);
//Face Rec on/off toggle
List <BumpSensorValidation> backRightBumpValidations = new List <BumpSensorValidation>();
backRightBumpValidations.Add(new BumpSensorValidation(BumpSensorFilter.SensorName, ComparisonOperator.Equal, BumpSensorPosition.BackRight));
backRightBumpValidations.Add(new BumpSensorValidation(BumpSensorFilter.IsContacted, ComparisonOperator.Equal, true));
_misty.RegisterBumpSensorEvent(ProcessBackRightBumpEvent, 250, true, backRightBumpValidations, null, null);
//Picture description
List <BumpSensorValidation> frontRightBumpValidations = new List <BumpSensorValidation>();
frontRightBumpValidations.Add(new BumpSensorValidation(BumpSensorFilter.SensorName, ComparisonOperator.Equal, BumpSensorPosition.FrontRight));
frontRightBumpValidations.Add(new BumpSensorValidation(BumpSensorFilter.IsContacted, ComparisonOperator.Equal, true));
_misty.RegisterBumpSensorEvent(ProcessFrontRightBumpEvent, 250, true, frontRightBumpValidations, null, null);
//Repeat
List <BumpSensorValidation> frontLeftBumpValidations = new List <BumpSensorValidation>();
frontLeftBumpValidations.Add(new BumpSensorValidation(BumpSensorFilter.SensorName, ComparisonOperator.Equal, BumpSensorPosition.FrontLeft));
frontLeftBumpValidations.Add(new BumpSensorValidation(BumpSensorFilter.IsContacted, ComparisonOperator.Equal, true));
_misty.RegisterBumpSensorEvent(ProcessFrontLeftBumpEvent, 250, true, frontLeftBumpValidations, null, null);
RegisterUserEvents();
if (!_misty.Wait(2000))
{
return;
}
//All ready to go
_assetWrapper.PlaySystemSound(SystemSound.Amazement);
_assetWrapper.ShowSystemImage(SystemImage.EcstacyStarryEyed);
_misty.ChangeLED(0, 0, 255, null);
if (!_misty.Wait(1500))
{
return;
}
BroadcastDetails("Hello! How are you?", _defaultVoice);
await _misty.TransitionLEDAsync(255, 0, 0, 0, 0, 255, LEDTransition.Breathe, 500);
_misty.Wait(1500);
await _misty.TransitionLEDAsync(0, 0, 255, 255, 140, 0, LEDTransition.Breathe, 1000);
/*
* //Test code to describe a random image at this URL
* BroadcastDetails("Describing random image.");
* _misty.Wait(500);
* string description = await _azureCognitive.AnalyzeImage("http://junglebiscuit.com/images/random/rand_image.pl");
* BroadcastDetails(description);
*/
}
catch (Exception ex)
{
_misty.SkillLogger.Log("Failed while starting the skill.", ex);
}
}
async void RunCustomVision()
{
try
{
_misty.SkillLogger.Log("Taking picture to analyze");
_misty.SendDebugMessage("Taking picture to analyze", null);
ITakePictureResponse takePictureResponse = await _misty.TakePictureAsync("oretest.jpg", false, true, true, 640, 480);
_misty.SendDebugMessage("Picture taken", null);
SoftwareBitmap softwareBitmap;
using (IRandomAccessStream stream = new MemoryStream((byte[])takePictureResponse.Data.Image).AsRandomAccessStream())
{
stream.Seek(0);
// Create the decoder from the stream
BitmapDecoder decoder = await BitmapDecoder.CreateAsync(stream);
// Get the SoftwareBitmap representation of the file in BGRA8 format
softwareBitmap = await decoder.GetSoftwareBitmapAsync();
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
}
// Encapsulate the image in the WinML image type (VideoFrame) to be bound and evaluated
VideoFrame inputImage = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
_misty.SendDebugMessage("Picture processed, sending to model", null);
// Evaluate the image
OnnxModelOutput output = await EvaluateVideoFrameAsync(inputImage);
_misty.SendDebugMessage("Model finished eval", null);
await _misty.DisplayImageAsync("e_DefaultContent.jpg", 100);
if (output == null)
{
_misty.SendDebugMessage("Model output empty", null);
_misty.ChangeLED(0, 0, 0, OnResponse);
alreadyRunning = false;
}
else
{
int vectorCount = output.detected_classes.GetAsVectorView().Count;
double initialScore = output.detected_scores.GetAsVectorView()[0];
long initialClass = output.detected_classes.GetAsVectorView()[0];
if (vectorCount == 0 || initialScore < 0.25)
{
_misty.ChangeLED(0, 0, 0, OnResponse);
alreadyRunning = false;
}
else if (initialClass == 1 && initialScore >= 0.25)
{
_misty.ChangeLED(255, 0, 0, OnResponse);
_misty.RunSkill("e1fcbf5b-9163-4d09-8707-bffd00ddcd5d", null, null);
alreadyRunning = false;
}
else if (initialClass == 0 && initialScore >= 0.25)
{
_misty.ChangeLED(0, 0, 255, OnResponse);
//Say found Ore
//_misty.RunSkill("a61832ab-6bc1-4f1a-9de1-0d1dc8bf3ff0", null, null);
var data = new StringContent("{ \"text\":\"Ore Found!\",\"pitch\":0,\"speechRate\":0,\"voice\":null,\"flush\":false,\"utteranceId\":null }", Encoding.UTF8, "application/json");
HttpResponseMessage result = await client.PostAsync("http://127.0.0.1/api/tts/speak?text=Ore Found!&pitch=0&speechRate=0&flush=false", data);
double calcTrajectory = yaw.getYaw() + (25 * (((output.detected_boxes.GetAsVectorView()[0] + output.detected_boxes.GetAsVectorView()[2]) / 2) - 0.5) * -1);
await _misty.SendDebugMessageAsync("Trajectory: " + calcTrajectory);
//Take the current yaw of the robot and then add the box X axis percentage
//The 20 number is approximately how many degrees you have to rotate to go from edge to center of the camera
if (calcTrajectory > yaw.getYaw())
{
await _misty.DriveAsync(0, 5);
}
else
{
await _misty.DriveAsync(0, -5);
}
//data = new StringContent("{ \"heading\":" + calcTrajectory.ToString() + ",\"radius\":0,\"timeMs\":3000,\"reverse\":false }", Encoding.UTF8, "application/json");
//result = await client.PostAsync("http://127.0.0.1/api/drive/arc", data);
yaw.setTargetYaw(calcTrajectory);
yaw.YawReached += HandleYawReached;
calcTrajectory = _currentHeadPitch + (80 * (((output.detected_boxes.GetAsVectorView()[1] + output.detected_boxes.GetAsVectorView()[3]) / 2) - 0.5));
await _misty.MoveHeadAsync(calcTrajectory, 0, 0, 100, AngularUnit.Degrees);
//_misty.DriveArc(calcTrajectory, 0.2, 2000, false, null);
//357.47 deg 50% at 2sec = 341.88 16 degree 342.46
}
}
}
catch (Exception ex)
{
alreadyRunning = false;
_misty.SendDebugMessage($"error: {ex.Message}", null);
_misty.SendDebugMessage("Picture processing failed", null);
}
}
/// <summary>
/// Called when the skill is cancelled
/// </summary>
public void OnCancel(object sender, IDictionary <string, object> parameters)
{
_misty.SkillLogger.LogVerbose($"LookAroundSkill : OnCancel called");
_misty.ChangeLED(255, 0, 0, null);
}