private static Dictionary <string, object> GenerateScoreKeywordCard(SkillResult result, dynamic boxBody)
{
var card = GetSkillCardTemplate(SkillType.keyword, boxBody, "Support Score", result.duration);
var entry = new Dictionary <string, object>()
{
{ "type", "text" },
{ "text", $"Value: {result.supportScore}" }
};
((List <Dictionary <string, object> >)card["entries"]).Add(entry);
if (result.supportScore <= 0m)
{
entry = new Dictionary <string, object>()
{
{ "type", "text" },
{ "text", "Followup: Negative" }
};
((List <Dictionary <string, object> >)card["entries"]).Add(entry);
}
else if (result.supportScore > 2.5m)
{
entry = new Dictionary <string, object>()
{
{ "type", "text" },
{ "text", "Followup: Positive" }
};
((List <Dictionary <string, object> >)card["entries"]).Add(entry);
}
return(card);
}
private static void CalculateSupportScore(ref SkillResult result)
{
var score = 1m;
foreach (var check in result.scriptChecks)
{
score += check.Value ? .5m : -.75m;
}
if (result.resultsBySpeakerSentiment[result.supportIndex].ContainsKey(NEGATIVE))
{
score += 1m - (0.05m * result.resultsBySpeakerSentiment[result.supportIndex][NEGATIVE].Count);
}
//TODO: bucketize across time and aggregate sentiment to see if moves from lower to higher (slope)
foreach (var tuple in result.resultsBySpeakerSentiment)
{
if (tuple.Key == result.supportIndex)
{
continue;
}
if (tuple.Value.ContainsKey(NEGATIVE))
{
score += 0m - (0.05m * tuple.Value[NEGATIVE].Count);
}
if (tuple.Value.ContainsKey(POSITIVE))
{
score += (0.1m * tuple.Value[POSITIVE].Count);
}
}
result.supportScore = score;
}
private static void AggregateSpeakerSentiment(ref SkillResult result)
{
foreach (var speaker in result.resultBySpeaker.Keys)
{
var sentimentAg = new Dictionary <string, List <SpeakerResult> >();
result.resultsBySpeakerSentiment.Add(speaker, new Dictionary <string, List <SpeakerResult> >());
foreach (var speakerResult in result.resultBySpeaker[speaker])
{
var sentString = speakerResult?.sentiment?.Sentiment?.Value;
//if (!sentString.Equals("NEUTRAL"))
if (sentString != null)
{
if (!sentimentAg.ContainsKey(sentString))
{
sentimentAg.Add(sentString, new List <SpeakerResult>());
}
sentimentAg[sentString].Add(speakerResult);
}
}
foreach (var sentValue in sentimentAg.Keys)
{
//capitalized first char
var formattedSentValue = sentValue.Remove(1) + sentValue.ToLower().Remove(0, 1);
result.resultsBySpeakerSentiment[speaker].Add(formattedSentValue, sentimentAg[sentValue]);
}
}
}
public static async Task GenerateCards(SkillResult result, dynamic boxBody)
{
var boxConfig = new BoxConfig(string.Empty, string.Empty, new Uri(config.BoxApiUrl));
var session = new OAuthSession(boxBody.token.write.access_token.Value, string.Empty, 3600, "bearer");
var client = new BoxClient(boxConfig, session);
if (client == null)
{
throw new Exception("Unable to create box client");
}
Console.WriteLine("======== BoxHelper Result =========");
Console.WriteLine(JsonConvert.SerializeObject(result, Formatting.None));
var cards = new List <Dictionary <string, object> >
{
GenerateSupportHeaderCard(result, boxBody),
GenerateScoreKeywordCard(result, boxBody),
GenerateTopicsKeywordCard(result, boxBody),
GenerateScriptAdherenceKeywordCard(result, boxBody),
GenerateTranscriptCard(result, boxBody)
};
cards.AddRange(GeneateSentimentTimelineCards(result, boxBody));
Console.WriteLine("======== Cards =========");
Console.WriteLine(JsonConvert.SerializeObject(cards, Formatting.None));
var skillsMetadata = new Dictionary <string, object>()
{
{ "cards", cards }
};
try {
await client.MetadataManager.CreateFileMetadataAsync(boxBody.source.id.Value, skillsMetadata, "global", "boxSkillsCards");
Console.WriteLine("Created metadata");
} catch (Exception e) {
Console.WriteLine("Exception creating metadata. Trying update");
Console.WriteLine(e);
BoxMetadataUpdate updateObj = new BoxMetadataUpdate
{
Op = MetadataUpdateOp.replace,
Path = "/cards",
Value = cards
};
try
{
await client.MetadataManager.UpdateFileMetadataAsync(boxBody.source.id.Value, new List <BoxMetadataUpdate>() { updateObj }, "global", "boxSkillsCards");
} catch (Exception e2) {
Console.WriteLine("Exception updating metadata. giving up");
Console.WriteLine(e2);
return;
}
Console.WriteLine("Successfully updated metadata");
}
}
public static void ProcessTranscriptionResults(ref SkillResult result)
{
GenerateScriptAdherence(ref result);
AggregateSpeakerSentiment(ref result);
AdjustSpeakerTitles(ref result);
CalculateSupportScore(ref result);
Console.WriteLine("======== SkillResult =========");
Console.WriteLine(JsonConvert.SerializeObject(result, Formatting.None));
}
private static void AdjustSpeakerTitles(ref SkillResult result)
{
result.speakerLabels[result.supportIndex] = "Support";
var spIdx = 1;
for (var i = 0; i < result.speakerLabels.Count; i++)
{
if (i != result.supportIndex)
{
result.speakerLabels[i] = "Customer" + (result.speakerLabels.Count == 2?"":$" {spIdx++}");
}
}
}
public static Dictionary <string, object> GenerateScriptAdherenceKeywordCard(SkillResult result, dynamic boxBody)
{
var card = GetSkillCardTemplate(SkillType.keyword, boxBody, "Script Adherence", result.duration);
foreach (var phraseKey in result.scriptChecks.Keys)
{
var entry = new Dictionary <string, object>()
{
{ "type", "text" },
{ "text", $"{phraseKey}: {result.scriptChecks[phraseKey]}" },
{ "appears", new List <Dictionary <string, object> >() }
};
((List <Dictionary <string, object> >)card["entries"]).Add(entry);
}
return(card);
}
public static Dictionary <string, object> GenerateSupportHeaderCard(SkillResult result, dynamic boxBody)
{
var card = GetSkillCardTemplate(SkillType.timeline, boxBody, "Support Representative", result.duration);
dynamic images = config.PeopleImages["support"];
string imageUrl = images[random.Next(0, images.Count - 1)].Value;
var entry = new Dictionary <string, object>()
{
{ "type", "image" },
{ "text", "Support Rep" },
{ "image_url", imageUrl }
};
((List <Dictionary <string, object> >)card["entries"]).Add(entry);
return(card);
}
private async Task <SkillResult> ProcessTranscriptionJob(TranscriptionJob finishedJob)
{
var result = new SkillResult();
if (finishedJob?.TranscriptionJobStatus.Value == JobStatus.FAILED)
{
Console.WriteLine($"Transcription job failed with reason: {finishedJob.FailureReason}");
}
else if (finishedJob.TranscriptionJobStatus.Value == JobStatus.COMPLETED)
{
Console.WriteLine($"Transcription file located @: {finishedJob.Transcript.TranscriptFileUri}");
var json = GetJobResultsForAnalsys(finishedJob.Transcript.TranscriptFileUri);
JObject transcriptionResults = JObject.Parse(json);
result = await ProcessTranscriptionResults(transcriptionResults);
}
return(result);
}
//Run through results, grouping words and saving the locations in the media file. Create card with top 20
//words more than 5 characters.
// TODO: should have common word list to ignore instead of <5 chars
// TODO: should calculate proximity to find phrases that appear together
public static Dictionary <string, object> GenerateTopicsKeywordCard(SkillResult result, dynamic boxBody)
{
var card = GetSkillCardTemplate(SkillType.keyword, boxBody, "Topics", result.duration);
var topics = new List <string>(result.topicLocations.Keys);
var count = 0;
topics.Sort(delegate(string a, string b)
{
return(result.topicLocations[a].Count.CompareTo(result.topicLocations[b].Count));
});
foreach (var topic in topics)
{
if (count++ == 20)
{
break;
}
var entry = new Dictionary <string, object>()
{
{ "type", "text" },
{ "text", topic },
{ "appears", new List <Dictionary <string, object> >() }
};
foreach (var speakerResult in result.topicLocations[topic])
{
var location = new Dictionary <string, object>()
{
{ "start", speakerResult.start },
{ "end", speakerResult.end }
};
((List <Dictionary <string, object> >)entry["appears"]).Add(location);
}
((List <Dictionary <string, object> >)card["entries"]).Add(entry);
}
return(card);
}
private static Dictionary <string, object> GenerateTranscriptCard(SkillResult result, dynamic boxBody)
{
var card = GetSkillCardTemplate(SkillType.transcript, boxBody, "Transcript", result.duration);
foreach (var speakerResult in result.resultByTime)
{
var entry = new Dictionary <string, object>()
{
{ "type", "text" },
{ "text", $"[{result.speakerLabels[speakerResult.speaker]}] {speakerResult.text}" },
{ "appears", new List <Dictionary <string, object> >()
{
new Dictionary <string, object>()
{
{ "start", speakerResult.start },
{ "end", speakerResult.end }
}
} }
};
((List <Dictionary <string, object> >)card["entries"]).Add(entry);
}
return(card);
}
private async Task <SkillResult> ProcessTranscriptionResults(JObject transcriptionResults)
{
var result = new SkillResult();
StringBuilder speakerText = new StringBuilder();
TranscribeAlternative alternative = null;
var segments = transcriptionResults["results"]["speaker_labels"]["segments"].ToObject <List <Segment> >();
var transciptionsItems = transcriptionResults["results"]["items"].ToObject <List <TranscribeItem> >();
Console.WriteLine($"items: {transciptionsItems?.Count} segments: {segments.Count}");
var speakerLabel = string.Empty;
var lastSpeaker = "nobody";
SpeakerResult currentSpeakerResult = new SpeakerResult();
var itemIdx = 0;
var ti = transciptionsItems;
// sements have a begin and end, however the items contained in it also
// have begin and ends. the range of the items have a 1 to 1 correlation to the 'pronunciation' transcription
// item types. These also have ends which are outside the range of the segement strangely. So will be using segment to
// get the speaker, then will create an inclusive range for all items under it using the being of first and end of last.
foreach (var segment in segments)
{
if (segment.items.Length == 0)
{
continue;
}
result.duration = segment.end_time;
if (!lastSpeaker.Equals(segment.speaker_label))
{
// these lines do nothing the first iteration, but tie up last
// speaker result when the speaker is changing
currentSpeakerResult.text = speakerText.ToString();
speakerText = new StringBuilder();
// create new speaker result for new speaker - or first speaker on first iteration
var idx = result.speakerLabels.IndexOf(segment.speaker_label);
if (idx == -1)
{
idx = result.speakerLabels.Count;
result.speakerLabels.Add(segment.speaker_label);
result.resultBySpeaker.Add(idx, new List <SpeakerResult>());
}
currentSpeakerResult = new SpeakerResult();
currentSpeakerResult.speaker = idx;
ConfigureTimeRange(ref currentSpeakerResult, segment);
lastSpeaker = segment.speaker_label;
result.resultBySpeaker[idx].Add(currentSpeakerResult);
result.resultByTime.Add(currentSpeakerResult);
}
else
{
ConfigureTimeRange(ref currentSpeakerResult, segment);
}
for (; itemIdx < ti.Count &&
((currentSpeakerResult.start <= ti[itemIdx].start_time && ti[itemIdx].end_time <= currentSpeakerResult.end) ||
(ti[itemIdx].start_time == 0m))
; itemIdx++)
{
alternative = ti[itemIdx].alternatives.First();
if (alternative.content.Equals("[SILENCE]"))
{
speakerText.Append(".");
}
else
{
speakerText.Append(alternative.content);
}
speakerText.Append(" ");
}
}
currentSpeakerResult.text = speakerText.ToString();
// Call AWS Comprehend client to get sentiment for all speaker results
List <int> keyList = new List <int>(result.resultBySpeaker.Keys);
for (int keyIdx = 0; keyIdx < keyList.Count; keyIdx++)
{
var spkKey = keyList[keyIdx];
for (int resultIdx = result.resultBySpeaker[spkKey].Count - 1; resultIdx >= 0; resultIdx--)
{
if (!IsBlankText(result.resultBySpeaker[spkKey][resultIdx].text))
{
var speakerResult = result.resultBySpeaker[spkKey][resultIdx];
speakerResult.sentiment = await DetermineSentiment(result.resultBySpeaker[spkKey][resultIdx].text);
var topics = await DetermineTopic(result.resultBySpeaker[spkKey][resultIdx].text);
foreach (var topic in topics)
{
if (!result.topicLocations.ContainsKey(topic.Text))
{
result.topicLocations.Add(topic.Text, new List <SpeakerResult>());
}
result.topicLocations[topic.Text].Add(speakerResult);
}
}
}
}
return(result);
}
public static List <Dictionary <string, object> > GeneateSentimentTimelineCards(SkillResult result, dynamic boxBody)
{
List <Dictionary <string, object> > cards = new List <Dictionary <string, object> >();
foreach (var speaker in result.resultsBySpeakerSentiment.Keys)
{
var card = GetSkillCardTemplate(SkillType.timeline, boxBody, $"{result.speakerLabels[speaker]} Sentiment", result.duration);
foreach (var sentValue in result.resultsBySpeakerSentiment[speaker].Keys)
{
dynamic image = config.SentimentImages[sentValue.ToLower()];
var entry = new Dictionary <string, object>()
{
{ "type", "text" },
{ "text", sentValue },
{ "image_url", image.Value },
{ "appears", new List <Dictionary <string, object> >() }
};
foreach (var speakerResult in result.resultsBySpeakerSentiment[speaker][sentValue])
{
var location = new Dictionary <string, object>()
{
{ "start", speakerResult.start },
{ "end", speakerResult.end }
};
((List <Dictionary <string, object> >)entry["appears"]).Add(location);
}
((List <Dictionary <string, object> >)card["entries"]).Add(entry);
}
cards.Add(card);
}
return(cards);
}
public static void GenerateScriptAdherence(ref SkillResult result)
{
//TODO: should use algorithm for symantic difference, but doing something quick for poc
// close text search
//we don't know which speaker is the support agent. So for each speaker, we will
//try to match all of the phrases. If we get more than 2 matches, we'll assume that
//is the support agent. Otherwise assume the first speaker and that they didn't
//follow the script
Console.WriteLine("======== Script Adherence =========");
Console.WriteLine(JsonConvert.SerializeObject(result, Formatting.None));
int lastPhraseCount = 0;
foreach (var speaker in result.resultBySpeaker.Keys)
{
var scriptChecks = new Dictionary <string, bool>();
var phraseCount = 0;
foreach (var phraseKey in scriptPhrases.Keys)
{
foreach (var results in result.resultBySpeaker[speaker])
{
var found = LevenshteinDistance.SearchPercent(scriptPhrases[phraseKey], CleanText(results.text));
//var found = CleanText(results.text).Contains(scriptPhrases[phraseKey]);
if (found >= .8m)
{
scriptChecks.Add(phraseKey, true);
break;
}
}
if (scriptChecks.ContainsKey(phraseKey))
{
phraseCount++;
}
else
{
scriptChecks.Add(phraseKey, false);
}
}
if (phraseCount > lastPhraseCount)
{
lastPhraseCount = phraseCount;
result.supportIndex = speaker;
result.scriptChecks = scriptChecks;
// With two matches, assume this has to be the support rep, so
// don't bother processing remaining speakers
if (phraseCount > 2)
{
break;
}
}
}
if (result.scriptChecks.Keys.Count == 0)
{
// no one matched anything, assume speaker one is support
result.supportIndex = 0;
foreach (var phraseKey in scriptPhrases.Keys)
{
result.scriptChecks.Add(phraseKey, false);
}
}
}
