// Use this for initialization
void Start()
{
currentEmotions = new EmotionStruct();
currentFACS = new FACSStruct();
emotionWindow = new ArrayList();
emotionWindow.Capacity = 10;
}
public string getStrongestEmotion2(EmotionStruct emotions)
{
int idx = -1;
string[] emotionNames = { "anger", "fear", "joy", "surprise", "sadness", "disgust" };
float[] emotionVals =
{
emotions.anger,
emotions.fear,
emotions.joy,
emotions.surprise,
emotions.sadness,
emotions.disgust
};
float max = float.MinValue;
for (int i = 0; i < emotionNames.Length; ++i)
{
if (emotionVals[i] > max)
{
max = emotionVals[i];
idx = i;
}
}
if (max > 10.0f)
{
return(emotionNames[idx]);
}
else
{
return("neutral");
}
}
void Start()
{
// Initialize the current emotions
currentEmotions = new EmotionStruct();
LogSystem.InstallDefaultReactors();
// Create credential and instantiate service
Credentials credentials = new Credentials(_username, _password, _url);
_speechToText = new SpeechToText(credentials);
Credentials naturalLanguageUnderstandingCredentials = new Credentials()
{
// Original Watson NLP credentials
// Username = "******",
// Password = "******",
Username = "******",
Password = "******",
Url = "https://gateway.watsonplatform.net/natural-language-understanding/api"
};
_nlu = new NaturalLanguageUnderstanding(naturalLanguageUnderstandingCredentials);
Active = true;
StartCoroutine(coroutineA());
StartRecording();
}
// Update is called once per frame
void Update()
{
// Pull in the most recent emotional state for each of the modalities
if (useFacialEmotion)
{
currentFacialEmotion = facialAnalyzer.getCurrentEmotions();
currentFacialEmotion2 = facialAnalyzer2.getCurrentEmotions();
currentFACS = facialAnalyzer.getCurrentFACS();
currentFACS2 = facialAnalyzer2.getCurrentFACS();
// Debug.Log("got facial emotion struct");
}
if (useWordSentimentEmotion)
{
currentWordSentimentEmotion = wordAnalyzer.getCurrentEmotions();
}
if (useVocalToneEmotion)
{
currentVocalEmotion = vocalAnalyzer.getVocalToneResults();
}
//Set mood tracker attributes
CalculateMoodTrackerGeometry();
}
// Use this for initialization
void Start()
{
// Initialize the current emotion
currentCumulativeEmotion = new EmotionStruct();
uiManagerScript = uIManager.GetComponent <UIManager> ();
// Find the script for facial emotion analysis
try {
facialAnalyzer = (FacialEmotionAnalyzer)facialEmotionAnalyzerObject.GetComponent(typeof(FacialEmotionAnalyzer)); // this seems to fail silently...
}
catch (System.Exception) {
Debug.Log("Unable to find facial emotion analyzer. This functionality will be disabled.");
useFacialEmotion = false;
}
}
// Returns a color to be used by the user interface based on the current synthesized emotion
public Color calculateEmotionColor(EmotionStruct emotions)
{
if (emotions.joy > emotions.fear &&
emotions.joy > emotions.disgust &&
emotions.joy > emotions.sadness &&
emotions.joy > emotions.anger &&
emotions.joy > emotions.surprise &&
emotions.joy > emotionThreshold)
{
return(new Color(0.0f, 1.0f, 0.0f, 1.0f)); // green
}
else if (emotions.fear > emotions.disgust &&
emotions.fear > emotions.sadness &&
emotions.fear > emotions.anger &&
emotions.fear > emotions.surprise &&
emotions.fear > emotionThreshold)
{
return(new Color(1.0f, 0.0f, 1.0f, 1.0f)); // magenta
}
else if (emotions.disgust > emotions.sadness &&
emotions.disgust > emotions.anger &&
emotions.disgust > emotions.surprise &&
emotions.disgust > emotionThreshold)
{
return(new Color(1.0f, 1.0f, 0.0f, 1.0f)); // yellow
}
else if (emotions.sadness > emotions.anger &&
emotions.sadness > emotions.surprise &&
emotions.sadness > emotionThreshold)
{
return(new Color(0.0f, 0.0f, 1.0f, 1.0f)); // blue
}
else if (emotions.anger > emotions.surprise &&
emotions.anger > emotionThreshold)
{
return(new Color(1.0f, 0.0f, 0.0f, 1.0f)); // red
}
else if (emotions.surprise > emotionThreshold)
{
return(new Color(1.0f, 1.0f, 1.0f, 1.0f)); // white
}
else
{
return(new Color(0.0f, 0.0f, 0.0f, 1.0f)); // black
}
}
// Returns the strongest emotion present in the current cumulative emotion
public string getValueOfStrongestEmotionString(EmotionStruct emotions)
{
if (emotions.joy > emotions.fear &&
emotions.joy > emotions.disgust &&
emotions.joy > emotions.sadness &&
emotions.joy > emotions.anger &&
emotions.joy > emotions.surprise &&
emotions.joy > emotionThreshold)
{
return("joy");
}
else if (emotions.fear > emotions.disgust &&
emotions.fear > emotions.sadness &&
emotions.fear > emotions.anger &&
emotions.fear > emotions.surprise &&
emotions.fear > emotionThreshold)
{
return("fear");
}
else if (emotions.disgust > emotions.sadness &&
emotions.disgust > emotions.anger &&
emotions.disgust > emotions.surprise &&
emotions.disgust > emotionThreshold)
{
return("disgust");
}
else if (emotions.sadness > emotions.anger &&
emotions.sadness > emotions.surprise &&
emotions.sadness > emotionThreshold)
{
return("sadness");
}
else if (emotions.anger > emotions.surprise &&
emotions.anger > emotionThreshold)
{
return("anger");
}
else if (emotions.surprise > emotionThreshold)
{
return("surprise");
}
else
{
return("neutral");
}
}
// Returns the strongest emotion present in the current cumulative emotion
public float getValueOfStrongestEmotion(EmotionStruct emotions)
{
if (emotions.joy > emotions.fear &&
emotions.joy > emotions.disgust &&
emotions.joy > emotions.sadness &&
emotions.joy > emotions.anger &&
emotions.joy > emotions.surprise &&
emotions.joy > emotionThreshold)
{
return(emotions.joy);
}
else if (emotions.fear > emotions.disgust &&
emotions.fear > emotions.sadness &&
emotions.fear > emotions.anger &&
emotions.fear > emotions.surprise &&
emotions.fear > emotionThreshold)
{
return(emotions.fear);
}
else if (emotions.disgust > emotions.sadness &&
emotions.disgust > emotions.anger &&
emotions.disgust > emotions.surprise &&
emotions.disgust > emotionThreshold)
{
return(emotions.disgust);
}
else if (emotions.sadness > emotions.anger &&
emotions.sadness > emotions.surprise &&
emotions.sadness > emotionThreshold)
{
return(emotions.sadness);
}
else if (emotions.anger > emotions.surprise &&
emotions.anger > emotionThreshold)
{
return(emotions.anger);
}
else if (emotions.surprise > emotionThreshold)
{
return(emotions.surprise);
}
else
{
return(0.0f);
}
}
// Sets the currentEmotions struct as an average of the past ten seconds worth of emotion collected from facial analysis.
private void calculateCurrentEmotion()
{
if (emotionWindow.Count > 0)
{
EmotionStruct emotionSum = new EmotionStruct();
foreach (EmotionStruct e in emotionWindow)
{
emotionSum.anger += e.anger;
emotionSum.joy += e.joy;
emotionSum.fear += e.fear;
emotionSum.sadness += e.sadness;
emotionSum.disgust += e.disgust;
emotionSum.surprise += e.surprise;
emotionSum.contempt += e.contempt;
emotionSum.valence += e.valence;
emotionSum.engagement += e.engagement;
}
currentEmotions.anger = emotionSum.anger / (float)emotionWindow.Count;
currentEmotions.joy = emotionSum.joy / (float)emotionWindow.Count;
currentEmotions.fear = emotionSum.fear / (float)emotionWindow.Count;
currentEmotions.sadness = emotionSum.sadness / (float)emotionWindow.Count;
currentEmotions.disgust = emotionSum.disgust / (float)emotionWindow.Count;
currentEmotions.surprise = emotionSum.surprise / (float)emotionWindow.Count;
currentEmotions.contempt = emotionSum.contempt / (float)emotionWindow.Count;
currentEmotions.valence = emotionSum.valence / (float)emotionWindow.Count;
currentEmotions.engagement = emotionSum.engagement / (float)emotionWindow.Count;
}
else
{
currentEmotions = new EmotionStruct();
}
// Debug.Log("emotion window count: " + emotionWindow.Count);
// Debug.Log("facial anger" + currentEmotions.anger);
// Debug.Log("facial joy" + currentEmotions.joy);
// Debug.Log("facial fear" + currentEmotions.fear);
// Debug.Log("facial sadness" + currentEmotions.sadness);
// Debug.Log("facial disgust" + currentEmotions.disgust);
// Debug.Log("facial surprise" + currentEmotions.surprise);
}
///////////////////////////////////////// SET/CALCULATE CUMULATIVE EMOTION START ////////////////////////////////////////////////////
// Returns a final emotion based on the emotion input types that are specified using a simple weighted average.
public void calculateCumulativeEmotion()
{
EmotionStruct emotionSum = new EmotionStruct();
int numEmotionModes = 0;
if (useFacialEmotion)
{
EmotionStruct facialEmotions = facialAnalyzer.getCurrentEmotions();
emotionSum.joy += facialEmotions.joy;
emotionSum.sadness += facialEmotions.sadness;
emotionSum.anger += facialEmotions.anger;
emotionSum.fear += facialEmotions.fear;
emotionSum.disgust += facialEmotions.disgust;
emotionSum.surprise += facialEmotions.surprise;
numEmotionModes++;
}
if (useWordSentimentEmotion)
{
Debug.Log("Need to implement emotion from word sentiment.");
}
if (useVocalToneEmotion)
{
Debug.Log("Need to implement emotion from vocal tone.");
}
if (numEmotionModes > 0)
{
currentCumulativeEmotion.joy = emotionSum.joy / (float)numEmotionModes;
currentCumulativeEmotion.sadness = emotionSum.sadness / (float)numEmotionModes;
currentCumulativeEmotion.anger = emotionSum.anger / (float)numEmotionModes;
currentCumulativeEmotion.fear = emotionSum.fear / (float)numEmotionModes;
currentCumulativeEmotion.disgust = emotionSum.disgust / (float)numEmotionModes;
currentCumulativeEmotion.surprise = emotionSum.surprise / (float)numEmotionModes;
}
}
public override void onImageResults(Dictionary <int, Face> faces)
{
// Debug.Log("Got face results");
foreach (KeyValuePair <int, Face> pair in faces)
{
int FaceId = pair.Key; // The Face Unique Id.
Face face = pair.Value; // Instance of the face class containing emotions, and facial expression values.
//Retrieve the Emotions Scores
// face.Emotions.TryGetValue(Emotions.Contempt, out currentContempt);
// face.Emotions.TryGetValue(Emotions.Valence, out currentValence);
// Ensure that emotion values are sampled only once a second, regardless of the frame rate
if (frameSampleCount % frameSampleRate == 0)
{
// new Thread(() =>
// {
// Thread.CurrentThread.IsBackground = true;
// /* run your code here */
// var bytes = File.ReadAllBytes(wavFile);
// var analysisResponseString = CreateWebRequest(analysisUrl, bytes, token);
// // Debug.Log(analysisResponseString);
// currentAnalysis = JSON.Parse(analysisResponseString);
// }).Start()
// Retrieve the emotion for this frame
EmotionStruct nextEmotion = new EmotionStruct();
face.Emotions.TryGetValue(Emotions.Joy, out nextEmotion.joy);
face.Emotions.TryGetValue(Emotions.Fear, out nextEmotion.fear);
face.Emotions.TryGetValue(Emotions.Disgust, out nextEmotion.disgust);
face.Emotions.TryGetValue(Emotions.Sadness, out nextEmotion.sadness);
face.Emotions.TryGetValue(Emotions.Anger, out nextEmotion.anger);
face.Emotions.TryGetValue(Emotions.Surprise, out nextEmotion.surprise);
face.Emotions.TryGetValue(Emotions.Contempt, out nextEmotion.contempt);
face.Emotions.TryGetValue(Emotions.Valence, out nextEmotion.valence);
face.Emotions.TryGetValue(Emotions.Engagement, out nextEmotion.engagement);
// Retrieve FACS values
face.Expressions.TryGetValue(Expressions.Smile, out currentFACS.Smile);
face.Expressions.TryGetValue(Expressions.InnerBrowRaise, out currentFACS.InnerEyeBrowRaise);
face.Expressions.TryGetValue(Expressions.BrowRaise, out currentFACS.BrowRaise);
face.Expressions.TryGetValue(Expressions.BrowFurrow, out currentFACS.BrowFurrow);
face.Expressions.TryGetValue(Expressions.NoseWrinkle, out currentFACS.NoseWrinkler);
face.Expressions.TryGetValue(Expressions.UpperLipRaise, out currentFACS.UpperLipRaiser);
face.Expressions.TryGetValue(Expressions.LipCornerDepressor, out currentFACS.LipCornerDepressor);
face.Expressions.TryGetValue(Expressions.ChinRaise, out currentFACS.ChinRaiser);
face.Expressions.TryGetValue(Expressions.LipPucker, out currentFACS.LipPucker);
face.Expressions.TryGetValue(Expressions.LipPress, out currentFACS.LipPress);
face.Expressions.TryGetValue(Expressions.LipSuck, out currentFACS.LipSuck);
face.Expressions.TryGetValue(Expressions.MouthOpen, out currentFACS.MouthOpen);
face.Expressions.TryGetValue(Expressions.Smirk, out currentFACS.Smirk);
face.Expressions.TryGetValue(Expressions.EyeClosure, out currentFACS.EyeClosure);
face.Expressions.TryGetValue(Expressions.Attention, out currentFACS.Attention);
//Not considering surprise or disgust for now
// nextEmotion.surprise = 0f;
// nextEmotion.disgust = 0f;
// currentEmotions = nextEmotion;
// Debug.Log("NEW ANGER!: " + nextEmotion.anger);
// Add in the next emotion captured by Affectiva
if (emotionWindow.Count == 4)
{
// add the tenth analysis to complete the window
emotionWindow.Add(nextEmotion);
// calculate the currentEmotions for this window and assign the parameter
calculateCurrentEmotion();
// shift the window back to prepare for the next emotion data
emotionWindow.RemoveAt(0);
}
else
{
// add the next element
emotionWindow.Add(nextEmotion);
}
}
frameSampleCount++;
// check for overflow and reset if it occurs
if (frameSampleCount < 0)
{
frameSampleCount = 0;
}
// Debug.Log("Frame sample count: " + frameSampleCount);
//Retrieve the Smile Score
// face.Expressions.TryGetValue(Expressions.Smile, out currentSmile);
//Retrieve the Interocular distance, the distance between two outer eye corners.
//Retrieve the coordinates of the facial landmarks (face feature points)
FeaturePoint[] featurePointsList = face.FeaturePoints;
Measurements measurementsList = face.Measurements;
moodTrackerParameters.x = featurePointsList [12].x;
moodTrackerParameters.y = featurePointsList [12].y;
moodTrackerParameters.z = measurementsList.interOcularDistance;
moodTrackerParameters.w = Mathf.Abs(featurePointsList [2].y - featurePointsList [12].y);
orientation = face.Measurements.Orientation;
}
}
private IEnumerator RequestEmotionUpdate()
{
// Debug.Log("Entered REQUEST EMOTION UPDATE COROUTINE.");
while (true)
{
yield return(new WaitForSeconds(colorUpdateTime));
if (gameManagerScript.useFacialEmotion)
{
EmotionStruct currentEmotions = gameManagerScript.getCurrentFacialEmotion();
EmotionStruct currentEmotions2 = gameManagerScript.getCurrentFacialEmotion2();
FACSStruct currentFACS = gameManagerScript.getCurrentFACS();
FACSStruct currentFACS2 = gameManagerScript.getCurrentFACS2();
FacialEmotionText.text = "Joy: " + currentEmotions.joy + "\nAnger: " + currentEmotions.anger + "\nFear: " + currentEmotions.fear + "\nDisgust: " + currentEmotions.disgust + "\nSadness: " + currentEmotions.sadness + "\nContempt: " + currentEmotions.contempt + "\nValence: " + currentEmotions.valence + "\nEngagement: " + currentEmotions.engagement;
FACSText.text = "Attention: " + currentFACS.Attention + "\nBrowFurrow: " + currentFACS.BrowFurrow + "\nBrowRaise: " + currentFACS.BrowRaise + "\nChinRaise: " + currentFACS.ChinRaiser + "\nEyeClose: " + currentFACS.EyeClosure + "\nInnerEyebrowRaise: " + currentFACS.InnerEyeBrowRaise + "\nLipCornerDepress: " + currentFACS.LipCornerDepressor + "\nLipPress: " + currentFACS.LipPress + "\nLipPucker: " + currentFACS.LipPucker + "\nLipSuck: " + currentFACS.LipSuck + "\nMouthOpen: " + currentFACS.MouthOpen + "\nNoseWrinkle: " + currentFACS.NoseWrinkler + "\nSmile: " + currentFACS.Smile + "\nSmirk: " + currentFACS.Smirk + "\nUpperLipRaise" + currentFACS.UpperLipRaiser;
// Update facial emotion colors
if (gameManagerScript.useAugmentedBasicEmotions)
{
currentValue = (currentEmotions.valence + 175f) / 300f;
currentHueDist = gameManagerScript.calculateEmotionHueDist(currentEmotions);
currentSaturation = sum(currentHueDist) / 100f;
currentHue = getStrongestHue(currentHueDist);
}
else if (gameManagerScript.useBasicEmotions)
{
}
else
{
currentValue = (currentEmotions.valence + 175f) / 300f;
currentSaturation = currentEmotions.engagement / 100f;
currentHue = (currentValue + currentSaturation) / 2;
}
string currentStrongestEmotion = getStrongestEmotion(currentHueDist);
string currentStrongestEmotion2 = getStrongestEmotion2(currentEmotions2);
int threshidx = 0;
if (getStrongestEmotionVal(currentHueDist) >= 50.0f)
{
threshidx = 1;
}
currentEmotionEmoji = facialEmojiDict [currentStrongestEmotion] [threshidx];
currentEmotionEmoji2 = facialEmojiDict [currentStrongestEmotion2] [threshidx];
if (currentStrongestEmotion == "joy")
{
rotationX = 0f;
}
else
{
rotationX = -90.0f;
}
if (currentStrongestEmotion2 == "joy")
{
rotationX2 = 0f;
}
else
{
rotationX2 = -90.0f;
}
previousFacialEmotionColor = currentFacialEmotionColor;
currentFacialEmotionColor = gameManagerScript.calculateEmotionColor(gameManagerScript.getCurrentFacialEmotion());
// Update the emotion bars
previousFacialEmotionBarWidth = currentFacialEmotionBarWidth;
currentFacialEmotionBarWidth = gameManagerScript.getValueOfStrongestEmotion(gameManagerScript.getCurrentFacialEmotion()) * 2;
}
if (gameManagerScript.useWordSentimentEmotion)
{
// Update word sentiment emotion colors
previousWordSentimentEmotionColor = currentWordSentimentEmotionColor;
currentWordSentimentEmotionColor = gameManagerScript.calculateEmotionColor(gameManagerScript.getCurrentWordSentimentEmotion());
EmotionStruct currentEmotions = gameManagerScript.getCurrentWordSentimentEmotion();
SentimentEmotionText.text = "Joy: " + currentEmotions.joy + "\nAnger: " + currentEmotions.anger + "\nFear: " + currentEmotions.fear + "\nDisgust: " + currentEmotions.disgust + "\nSadness: " + currentEmotions.sadness;
previousWordSentimentEmotionBarWidth = currentWordSentimentEmotionBarWidth;
currentWordSentimentEmotionBarWidth = gameManagerScript.getValueOfStrongestEmotion(gameManagerScript.getCurrentWordSentimentEmotion()) * 2;
}
if (gameManagerScript.useVocalToneEmotion)
{
if (gameManagerScript.useFacialEmotion)
{
currentSaturation += vocalToneResults.ArousalVal / 100f;
currentValue += vocalToneResults.ValenceVal / 100f;
currentSaturation /= 2f;
currentValue /= 2f;
}
else
{
currentSaturation = vocalToneResults.ArousalVal / 100f;
currentValue = vocalToneResults.ValenceVal / 100f;
}
}
currentBackground = Color.HSVToRGB(currentHue, currentSaturation, currentValue);
StartCoroutine(UpdateBackgroundColor());
}
}
// Use this for initialization
void Start()
{
currentEmotions = new EmotionStruct();
}
public float[] calculateEmotionHueDist(EmotionStruct emotions)
{
float[] hueDist = { emotions.anger, emotions.fear, emotions.joy, emotions.surprise, emotions.sadness, emotions.disgust };
return(hueDist);
}
