public async Task <List <KeyPhrase> > DetermineTopic(string text)
{
DetectKeyPhrasesRequest request = new DetectKeyPhrasesRequest()
{
LanguageCode = "en",
Text = text
};
DetectKeyPhrasesResponse response = await config.AzComprehendClient.DetectKeyPhrasesAsync(request);
var result = new List <KeyPhrase>();
// Filter using a score of .9 as the threshold
if (response != null && response.KeyPhrases.Count > 0)
{
foreach (var phrase in response.KeyPhrases)
{
if (phrase.Score > -.7 && phrase.Text.Length > 4)
{
Console.WriteLine($"=== Phrase [{phrase.Text}] score: {phrase.Score}");
var str = phrase.Text;
phrase.Text = str.Remove(1).ToUpper() + str.Remove(0, 1).ToLower();
result.Add(phrase);
}
}
}
return(result);
}
private List <string> DetectKeyPhrases(string text)
{
List <string> keyPhrases = new List <string>();
if (text.Length > 2500)
{
text = text.Substring(0, 2499);
}
AmazonComprehendClient comprehendClient = new AmazonComprehendClient(AWS_KEY, AWS_SECRET, Amazon.RegionEndpoint.USWest2);
// Call DetectKeyPhrases API
Console.WriteLine("Calling DetectKeyPhrases");
DetectKeyPhrasesRequest detectKeyPhrasesRequest = new DetectKeyPhrasesRequest()
{
Text = text,
LanguageCode = "en"
};
DetectKeyPhrasesResponse detectKeyPhrasesResponse = comprehendClient.DetectKeyPhrasesAsync(detectKeyPhrasesRequest).Result;
if (detectKeyPhrasesResponse.HttpStatusCode == System.Net.HttpStatusCode.OK && detectKeyPhrasesResponse.KeyPhrases != null)
{
keyPhrases = (from kp in detectKeyPhrasesResponse.KeyPhrases
where kp.Score >= MINIMUM_THRESHOLD_SCORE
select kp.Text).ToList();
}
return(keyPhrases);
}
static void Main(string[] args)
{
// Log into client with credentials
AmazonComprehendClient client = new AmazonComprehendClient(Amazon.RegionEndpoint.USEast1);
// Load XML doc
XmlDocument xmlDoc = new XmlDocument();
xmlDoc.Load("phrases.xml");
XmlElement rootElement = xmlDoc.DocumentElement;
XmlNodeList rootsChildren = rootElement.ChildNodes;
foreach (XmlNode node in rootsChildren)
{
DetectKeyPhrasesRequest request = new DetectKeyPhrasesRequest
{
Text = node.InnerText,
LanguageCode = "en"
};
DetectKeyPhrasesResponse response = client.DetectKeyPhrases(request);
Console.WriteLine(node.InnerText);
foreach (KeyPhrase kp in response.KeyPhrases)
{
Console.Write("\t" + kp.Text + " (" + kp.Score + ") \n");
}
Console.WriteLine();
}
Console.ReadLine();
}
private void CreateKeyPhraseCSV(string key, DetectKeyPhrasesResponse keyPhrasesResponse)
{
try
{
using (MemoryStream memStream = new MemoryStream())
{
var csv = new CsvWriter(new StreamWriter(memStream, Encoding.UTF8));
foreach (var a in keyPhrasesResponse.KeyPhrases)
{
dynamic keyPhrase = new System.Dynamic.ExpandoObject();
keyPhrase.Id = key.Replace(".json", "");
keyPhrase.Score = a.Score;
keyPhrase.Text = a.Text;
csv.WriteRecord <dynamic>(keyPhrase);
csv.NextRecord();
csv.Flush();
}
SendFileToS3(memStream, @"athena-data/key-phrases", key.Replace(".json", ".csv"));
}
}
catch (Exception ex)
{
var sdf = "test";
}
}
private void getKeyPhrases(string mailbody)
{
const string collectionUri = "https://dev.azure.com/codeletehackathon";
const string projectName = "secondproject";
const string repoName = "secondproject";
const string pat = "4rm4d2q2wq452n7ce2mqgfj4yyqajoipmez63nvxoveijuphckxq";
var creds = new VssBasicCredential(string.Empty, pat);
// Connect to Azure DevOps Services
var connection = new VssConnection(new Uri(collectionUri), creds);
// Get a GitHttpClient to talk to the Git endpoints
var gitClient = connection.GetClient <GitHttpClient>();
// Get data about a specific repository
var repo = gitClient.GetRepositoryAsync(projectName, repoName).Result;
var repoid = repo.Id;
string collectionuris = "https://dev.azure.com/codeletehackathon/secondproject/_apis/git/repositories/506f6d44-0a42-4a10-9f75-c89c450c3740/commits?api-version=6.0";
//String text = "Thank you for helping your project manager, Angela Parmar, on the latest project. Your work reflects your dedication to the project and the organisation.";
AmazonComprehendClient comprehendClient = new AmazonComprehendClient(Amazon.RegionEndpoint.USWest2);
// Call DetectKeyPhrases API
Console.WriteLine("Calling DetectKeyPhrases");
DetectKeyPhrasesRequest detectKeyPhrasesRequest = new DetectKeyPhrasesRequest()
{
Text = mailbody,
LanguageCode = "en"
};
DetectKeyPhrasesResponse detectKeyPhrasesResponse = comprehendClient.DetectKeyPhrases(detectKeyPhrasesRequest);
DetectSyntaxRequest dsr;
DetectSyntaxResponse dsresp;
List <string> lkeyPhrases = new List <string>();
string strKeyPhrases = string.Empty;
foreach (KeyPhrase kp in detectKeyPhrasesResponse.KeyPhrases)
{
lkeyPhrases.Add(kp.Text);
strKeyPhrases = strKeyPhrases + ',' + kp.Text;
}
var values = new object[2];
values[0] = "Key Phrases";
values[1] = strKeyPhrases.Substring(1);
dtTextAnalytic.Rows.Add(values);
Console.WriteLine("Done");
}
/// <summary>
/// Unmarshaller the response from the service to the response class.
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
DetectKeyPhrasesResponse response = new DetectKeyPhrasesResponse();
context.Read();
int targetDepth = context.CurrentDepth;
while (context.ReadAtDepth(targetDepth))
{
if (context.TestExpression("KeyPhrases", targetDepth))
{
var unmarshaller = new ListUnmarshaller <KeyPhrase, KeyPhraseUnmarshaller>(KeyPhraseUnmarshaller.Instance);
response.KeyPhrases = unmarshaller.Unmarshall(context);
continue;
}
}
return(response);
}
private async Task <string> GetComprehendData(string transcriptText, string key)
{
JObject transcriptJSON = JObject.Parse(transcriptText);
string test = (string)transcriptJSON["Text"];
AmazonComprehendClient client = new AmazonComprehendClient();
DetectEntitiesRequest entitiesRequest = new DetectEntitiesRequest();
entitiesRequest.LanguageCode = LanguageCode.En;
entitiesRequest.Text = test;
DetectSentimentRequest sentimentRequest = new DetectSentimentRequest();
sentimentRequest.LanguageCode = LanguageCode.En;
sentimentRequest.Text = test;
DetectKeyPhrasesRequest keyPhrasesRequest = new DetectKeyPhrasesRequest();
keyPhrasesRequest.LanguageCode = LanguageCode.En;
keyPhrasesRequest.Text = test;
DetectEntitiesResponse entitiesResponse = await client.DetectEntitiesAsync(entitiesRequest);
DetectSentimentResponse setimentResponse = await client.DetectSentimentAsync(sentimentRequest);
DetectKeyPhrasesResponse keyPhrasesResponse = await client.DetectKeyPhrasesAsync(keyPhrasesRequest);
CreateKeyPhraseCSV(key, keyPhrasesResponse);
CreateSetimentJSON(key, setimentResponse);
//now send the file to s3
//we need to write two different files, one for setiment and one for Key Phrases.
return(string.Empty);
}
