/// <summary>
/// Adds found key phrases to references list of keyPhrases - the most important ones first.
/// </summary>
/// <param name="keyPhrases"></param>
/// <param name="analyzable"></param>
/// <param name="log"></param>
private static void RunTextAnalysis(ref List <string> keyPhrases, List <MultiLanguageInput> analyzable, TraceWriter log)
{
var batch = new MultiLanguageBatchInput();
batch.Documents = analyzable;
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Westus;
client.SubscriptionKey = key1;
try
{
var result = client.KeyPhrases(batch);
foreach (var row in result.Documents)
{
foreach (var kp in row.KeyPhrases)
{
keyPhrases.Add(kp);
}
}
}
catch (Exception ex)
{
log.Warning(ex.Message);
}
}
public static List <string> getAnalitcsResult(string inputText)
{
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Westcentralus;
client.SubscriptionKey = "";
List <string> strList = new List <string>();
KeyPhraseBatchResult result2 = client.KeyPhrases(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput("en", "1", inputText)
}));
strList.Add(result2.Documents[0].KeyPhrases[0]);
SentimentBatchResult result3 = client.Sentiment(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput("en", "1", inputText)
}));
strList.Add(result3.Documents[0].Score.ToString());
return(strList);
}
/// <summary>
/// Query the Azure congnitive analytics API for text analysis. This utilizes a nuget package and is largely the example
/// code they provided with a few tweaks.
/// </summary>
/// <param name="body">Text body to be analyzed</param>
/// <returns>an instance of the Analytics class that has the relevant parts of the analysis repackaged for ease of use</returns>
public Analytics Analyze(string body)//Note: this was private, but that made testing the controller difficult
{
// Create a client.
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Westcentralus;
client.SubscriptionKey = Configuration["textAPIKey"];
// initialize output vars
List <string> keyPhrases = new List <string>();
float sentiment = 0;
Console.OutputEncoding = System.Text.Encoding.UTF8;
// Getting key-phrases
// this is taken almost word for word from the example code in the docs for the API
KeyPhraseBatchResult result2 = client.KeyPhrases(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput("en", "3", body),
}));
// Unpack key phrases into list
foreach (var document in result2.Documents)
{
foreach (string keyphrase in document.KeyPhrases)
{
keyPhrases.Add(keyphrase);
}
}
// Extracting sentiment
SentimentBatchResult result3 = client.Sentiment(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput("en", "0", body)
}));
// Unpack sentiment results
foreach (var document in result3.Documents)
{
sentiment = (float)document.Score;
}
// Repack analysis into analytics instance for convenience of use.
return(new Analytics()
{
Keywords = keyPhrases, Sentiment = sentiment
});
}
/// <summary>
/// Analyzes all the given sentences and queries them against the different apis
/// </summary>
/// <param name="origin"></param>
/// <param name="URLsources"></param>
/// <param name="Sources"></param>
/// <returns></returns>
public List <Plagiat <string> > Check(List <string> origin, List <Uri> URLsources, List <String> Sources)
{
//google search with custom search
GoogleSearch googleSearch = new GoogleSearch(new Uri("https://www.googleapis.com/customsearch/v1"));
//europa pmc search for academic literatur in life science
EuropaPMCSearch europaPMCSearch = new EuropaPMCSearch(new Uri("https://www.ebi.ac.uk/europepmc/webservices/rest/search?"));
//create document statistics
DocumentStatistics documentStatistics = new DocumentStatistics(origin);
googleSearch.Check(documentStatistics);
europaPMCSearch.Check(documentStatistics);
//starting azure congitive services to interpret sentence
// Create a client
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Westeurope;
client.SubscriptionKey = "<placekey>";
// Extracting language
LanguageBatchResult languagesDetected = client.DetectLanguage(
new BatchInput(documentStatistics.getBatchInput())
);
//store results
foreach (var document in languagesDetected.Documents)
{
documentStatistics.updateSentenceLanguage(document.Id, document.DetectedLanguages[0].Iso6391Name);
}
// Getting key-phrases
KeyPhraseBatchResult keyPhares = client.KeyPhrases(
new MultiLanguageBatchInput(documentStatistics.getMultiLanguageBatchInput())
);
// Printing keyphrases
foreach (var document in keyPhares.Documents)
{
documentStatistics.updateKeyPhares(document.Id, (List <string>)document.KeyPhrases);
}
return(documentStatistics.getPossiblePlagiates());
}
public KeyPhraseBatchResult GetKeyPhrases()
{
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Westeurope;
client.SubscriptionKey = "";
KeyPhraseBatchResult result2 = client.KeyPhrases(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput("ja", "1", "猫は幸せ"),
new MultiLanguageInput("de", "2", "Fahrt nach Stuttgart und dann zum Hotel zu Fu."),
new MultiLanguageInput("en", "3", "My cat is stiff as a rock."),
new MultiLanguageInput("es", "4", "A mi me encanta el fútbol!")
}));
return(result2);
}
public KeyPhraseBatchResult GetKeyPhraseList(IList <LanguageModel> list)
{
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Westeurope;
client.SubscriptionKey = "";
var listInput = new List <MultiLanguageInput>();
for (int i = 0; i < list.Count; i++)
{
var language = DetectLanguageServiceForAString(list[i].Text).Documents[0].DetectedLanguages[0].Iso6391Name;
var input = new MultiLanguageInput(list[i].Language, i.ToString(), list[i].Text);
listInput.Add(input);
}
KeyPhraseBatchResult result2 = client.KeyPhrases(
new MultiLanguageBatchInput(listInput));
return(result2);
}
static IList <string> ExtractKeyPhrases(string content)
{
// Create a client.
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Westus;
client.SubscriptionKey = TextAnalyticsAPIKey;
Console.OutputEncoding = System.Text.Encoding.UTF8;
// Getting key-phrases
KeyPhraseBatchResult result2 = client.KeyPhrases(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput("en", "1", content)
}));
// Since I am only sending one document, I can return just the first one
return(result2.Documents[0].KeyPhrases);
}
private async Task MessageReceivedAsync(IDialogContext context, IAwaitable <object> result)
{
var activity = await result as Activity;
// Create a client.
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Westeurope;
client.SubscriptionKey = Environment.GetEnvironmentVariable("clientSubscriptionKey");
_inputCount++;
//Luis
var httpClient = new HttpClient();
var queryString = HttpUtility.ParseQueryString(string.Empty);
var luisAppId = Environment.GetEnvironmentVariable("luisAppId");
var subscriptionKey = Environment.GetEnvironmentVariable("subscriptionKey");
// The request header contains your subscription key
httpClient.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", subscriptionKey);
// The "q" parameter contains the utterance to send to LUIS
queryString["q"] = activity.Text;
// These optional request parameters are set to their default values
queryString["timezoneOffset"] = "0";
queryString["verbose"] = "false";
queryString["spellCheck"] = "false";
queryString["staging"] = "false";
var uri = "https://westeurope.api.cognitive.microsoft.com/luis/v2.0/apps/" + luisAppId + "?" + queryString;
var response = await httpClient.GetAsync(uri);
string lang = null;
var dataFromLuis = JsonConvert.DeserializeObject <LuisResponse>(response.Content.ReadAsStringAsync().Result);
if (dataFromLuis.entities.Length > 0)
{
lang = dataFromLuis.entities[0].entity;
}
//Finne språket
LanguageBatchResult res = client.DetectLanguage(
new BatchInput(
new List <Input>
{
new Input(_inputCount.ToString(), activity.Text)
}));
// calculate something for us to return
int length = (activity.Text ?? string.Empty).Length;
StringBuilder keyWordBuilder = new StringBuilder();
keyWordBuilder.Append(" ");
// Printing language results.
foreach (var document in res.Documents)
{
//Finne nøkkelfraser
KeyPhraseBatchResult res2 = client.KeyPhrases(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>
{
new MultiLanguageInput(document.DetectedLanguages[0].Iso6391Name, _inputCount.ToString(),
activity.Text)
}));
// Printing keyphrases
foreach (var doc2 in res2.Documents)
{
foreach (string keyphrase in doc2.KeyPhrases)
{
keyWordBuilder.Append(keyphrase + " ");
}
if (doc2.KeyPhrases.Count == 0)
{
keyWordBuilder.Append("Fant ingen nøkkelfraser");
}
}
// Extracting sentiment
SentimentBatchResult res3 = client.Sentiment(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>
{
new MultiLanguageInput(document.DetectedLanguages[0].Iso6391Name, _inputCount.ToString(),
activity.Text)
}));
// Printing sentiment results
foreach (var doc3 in res3.Documents)
{
ConsultantResponse dataFromResponsefromConsultant = null;
var httpConsultant = new HttpClient();
httpConsultant
.DefaultRequestHeaders
.Accept
.Add(new MediaTypeWithQualityHeaderValue("application/json")); //ACCEPT header
if (lang != null)
{
var responsefromConsultant =
await httpConsultant.GetAsync(
$"http://37.139.15.166/consultants?skill={dataFromLuis.entities[0].entity}");
dataFromResponsefromConsultant =
JsonConvert.DeserializeObject <ConsultantResponse>(
responsefromConsultant.Content.ReadAsStringAsync().Result);
}
string returnConsultantIfData(ConsultantResponse cr)
{
int count = 0;
if (cr != null && cr.consultants.Length > 0)
{
StringBuilder cnBuilder = new StringBuilder();
cnBuilder.AppendLine(
$"I hear you are looking for people that know {dataFromLuis.entities[0].entity} programming language. In our resource database i found: ");
foreach (var c in cr.consultants)
{
cnBuilder.Append(c.name);
count++;
if (count < cr.consultants.Length)
{
cnBuilder.Append(", ");
}
}
return(cnBuilder.ToString());
}
return(null);
}
var textInput = activity.Text;
var langIs = document.DetectedLanguages[0].Name;
var keyFrases = keyWordBuilder.ToString().TrimEnd();
var emotionScoreIs = $"{doc3.Score:0.00}";
bool onlyLuisApi = true;
StringBuilder responsBuilder = new StringBuilder();
if (onlyLuisApi)
{
//Only luis fetch programming skills
responsBuilder.
Append(returnConsultantIfData(dataFromResponsefromConsultant));
}
else
{
//With detect language, sentiment, key frases and luis programming skills
responsBuilder.
Append("Hello! You wrote ").
AppendLine(textInput + ".").
Append("The language is most likely: ").
AppendLine(langIs + ".").
Append("The key frases are: ").
AppendLine(keyFrases + ".").
Append("Based what you wrote i detected the sentiment score: ").
AppendLine(emotionScoreIs + " On a scale between 0-1, where 0 is the most negative(sad) and 1 is most positive(happy).").
Append(returnConsultantIfData(dataFromResponsefromConsultant));
}
// return our reply to the user
if (responsBuilder.Length > 0)
{
await context.PostAsync(responsBuilder.ToString());
}
}
}
context.Wait(MessageReceivedAsync);
}
static void Main(string[] args)
{
// Create a client.
ITextAnalyticsAPI client = new TextAnalyticsAPI
{
AzureRegion = AzureRegions.Westus,
SubscriptionKey = "your subscription key"
};
Console.OutputEncoding = System.Text.Encoding.UTF8;
// Extracting language
Console.WriteLine("===== LANGUAGE EXTRACTION ======");
LanguageBatchResult result = client.DetectLanguage(
new BatchInput(
new List <Input>()
{
new Input("1", "This is a document written in English."),
new Input("2", "Este es un document escrito en Español."),
new Input("3", "这是一个用中文写的文件")
}));
// Printing language results.
foreach (var document in result.Documents)
{
Console.WriteLine("Document ID: {0} , Language: {1}", document.Id, document.DetectedLanguages[0].Name);
}
// Getting key-phrases
Console.WriteLine("\n\n===== KEY-PHRASE EXTRACTION ======");
KeyPhraseBatchResult result2 = client.KeyPhrases(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput("ja", "1", "猫は幸せ"),
new MultiLanguageInput("de", "2", "Fahrt nach Stuttgart und dann zum Hotel zu Fu."),
new MultiLanguageInput("en", "3", "My cat is stiff as a rock."),
new MultiLanguageInput("es", "4", "A mi me encanta el fútbol!")
}));
// Printing keyphrases
foreach (var document in result2.Documents)
{
Console.WriteLine("Document ID: {0} ", document.Id);
Console.WriteLine("\t Key phrases:");
foreach (string keyphrase in document.KeyPhrases)
{
Console.WriteLine("\t\t" + keyphrase);
}
}
// Extracting sentiment
Console.WriteLine("\n\n===== SENTIMENT ANALYSIS ======");
SentimentBatchResult result3 = client.Sentiment(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput("en", "0", "I had the best day of my life."),
new MultiLanguageInput("en", "1", "This was a waste of my time. The speaker put me to sleep."),
new MultiLanguageInput("es", "2", "No tengo dinero ni nada que dar..."),
new MultiLanguageInput("it", "3", "L'hotel veneziano era meraviglioso. È un bellissimo pezzo di architettura."),
}));
// Printing sentiment results
foreach (var document in result3.Documents)
{
Console.WriteLine("Document ID: {0} , Sentiment Score: {1:0.00}", document.Id, document.Score);
}
}
public static StoryLine createStory(string userInput)
{
// Create a client.
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Westus;
client.SubscriptionKey = "49bd3a3a1a244fd289aa30b7a5594b05";
Console.OutputEncoding = System.Text.Encoding.UTF8;
// Extracting language
Console.WriteLine("===== LANGUAGE EXTRACTION ======");
// CALLING TAMMY's FUNCTION TO GET THE USER INPUT STRING
string inputString = "I live in HongKong for nine years now. After that, I went to UCSD for college. I miss food from hometown.";
// Split each line according to period. Afterr the speech ends, return an empty string.
string [] singleLine = inputString.Split('.');
string [] keyWordResult = new string[singleLine.Length];
double [] sentimentResult = new double[singleLine.Length];
List <Input> inputLine = new List <Input>();
int count = 0;
foreach (var line in singleLine)
{
//Console.WriteLine($"<{line}>");
inputLine.Add(new Input(count.ToString(), line));
count++;
}
string[] languages = new string[inputLine.Count];
LanguageBatchResult result = client.DetectLanguage(
new BatchInput(
inputLine
));
// Updating language results.
count = 0;
foreach (var document in result.Documents)
{
//Console.WriteLine("Document ID: {0} , Language: {1}", document.Id, document.DetectedLanguages[0].Iso6391Name);
languages[count] = document.DetectedLanguages[0].Iso6391Name;
count++;
}
// Getting key-phrases
Console.WriteLine("\n\n===== KEY-PHRASE EXTRACTION ======");
int languageCount = 0;
count = 0;
List <MultiLanguageInput> inputKeywordLine = new List <MultiLanguageInput>();
foreach (var key in singleLine)
{
//Console.WriteLine("The language is: {0}, The count is {1}, the key is {2}", languages[languageCount], count.ToString(),key);
inputKeywordLine.Add(new MultiLanguageInput(languages[languageCount], count.ToString(), key));
count++;
}
KeyPhraseBatchResult result2 = client.KeyPhrases(
new MultiLanguageBatchInput(
inputKeywordLine
));
// Printing keyphrases
foreach (var document in result2.Documents)
{
//Console.WriteLine("Document ID: {0} ", document.Id);
//Console.WriteLine("\t Key phrases: {0}", document.KeyPhrases[0]);
keyWordResult[Int32.Parse(document.Id)] = document.KeyPhrases[0];
//Console.WriteLine(keyWordResult[Int32.Parse(document.Id)]);
/*
* foreach (string keyphrase in document.KeyPhrases)
* {
* Console.WriteLine("\t\t" + keyphrase);
* }
*/
}
// Extracting sentiment
Console.WriteLine("\n\n===== SENTIMENT ANALYSIS ======");
SentimentBatchResult result3 = client.Sentiment(
new MultiLanguageBatchInput(
inputKeywordLine
/*
* new List<MultiLanguageInput>()
* {
* new MultiLanguageInput("en", "1", "I live in HongKong for nine years now."),
* new MultiLanguageInput("en", "2", "After that, I went to UCSD for college."),
* new MultiLanguageInput("en", "3", " I miss food from hometown."),
* }*/
));
// Printing sentiment results
foreach (var document in result3.Documents)
{
sentimentResult[Int32.Parse(document.Id)] = Convert.ToDouble(document.Score);
//Console.WriteLine(sentimentResult[Int32.Parse(document.Id)]);
}
return(new StoryLine(languages, keyWordResult, sentimentResult));
}
static void Main(string[] args)
{
//init database configuration
string cmdString = "INSERT INTO Review (reviewerID, asin, reviewerName,helpful, not_helpful, reviewText,overall,summary,unixReviewTime,reviewTime,KeyPhrases) " +
"VALUES (@reviewerID, @asin, @reviewerName, @helpful, @not_helpful, @reviewText,@overall,@summary,@unixReviewTime,@reviewTime,@KeyPhrases)";
string connection = "Server=pbi-customerreivew.database.windows.net;Database=customerreview;User Id=louisli;Password = longP@$$w0rd1; ";
//init Cognitive API
// Create a client.
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Westcentralus;
client.SubscriptionKey = "<Get Key from Azure Text Analytics API and paste it here>";
//Init Event Hub
var connectionStringBuilder = new EventHubsConnectionStringBuilder(EhConnectionString)
{
EntityPath = EhEntityPath
};
eventHubClient = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());
//Start loading reviews
using (SqlConnection conn = new SqlConnection(connection))
{
conn.Open();
foreach (string line in File.ReadLines(@"D:\Training\reviews_Musical_Instruments_5.json\Musical_Instruments_5_reduced.json"))
{
//Read one record from JSON file
ReviewObject review = JsonConvert.DeserializeObject <ReviewObject>(line);
// Get key words
KeyPhraseBatchResult result2 = client.KeyPhrases(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput("en", "1", review.reviewText),
new MultiLanguageInput("en", "2", review.summary)
}));
foreach (var document in result2.Documents)
{
foreach (string keyphrase in document.KeyPhrases)
{
review.KeyPhrases += keyphrase + ",";
}
}
review.KeyPhrases = review.KeyPhrases.Substring(0, review.KeyPhrases.Length - 1);
//Load into database
using (SqlCommand comm = new SqlCommand())
{
comm.Connection = conn;
comm.CommandText = cmdString;
comm.Parameters.AddWithValue("@reviewerID", review.reviewerID);
comm.Parameters.AddWithValue("@asin", review.asin);
comm.Parameters.AddWithValue("@reviewerName", review.reviewerName);
comm.Parameters.AddWithValue("@helpful", review.helpful[0]);
comm.Parameters.AddWithValue("@not_helpful", review.helpful[1]);
comm.Parameters.AddWithValue("@reviewText", review.reviewText);
comm.Parameters.AddWithValue("@overall", review.overall);
comm.Parameters.AddWithValue("@summary", review.summary);
comm.Parameters.AddWithValue("@unixReviewTime", review.unixReviewTime);
comm.Parameters.AddWithValue("@reviewTime", review.reviewTime);
comm.Parameters.AddWithValue("@KeyPhrases", review.KeyPhrases);
try
{
comm.ExecuteNonQuery();
}
catch (SqlException e)
{
// do something with the exception
// don't hide it
}
}
//Send review to Event Hub
EventData msg = new EventData(Encoding.UTF8.GetBytes(review.ToJson()));
eventHubClient.SendAsync(msg);
Thread.Sleep(3000);
}
}
}
public string Get(string textToInspect)
{
// Create a client.
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Westeurope;
client.SubscriptionKey = config["TextAnalysisKey"];
StringBuilder sb = new StringBuilder();
// Extracting language
sb.AppendLine("===== LANGUAGE EXTRACTION ======");
LanguageBatchResult result = client.DetectLanguage(
new BatchInput(
new List <Input>()
{
new Input("1", textToInspect),
}));
// Printing language results.
foreach (var document in result.Documents)
{
sb.AppendLine($"Document ID: {document.Id} , Language: {document.DetectedLanguages[0].Name}");
// Getting key-phrases
sb.AppendLine("\n\n===== KEY-PHRASE EXTRACTION ======");
var isoLanguageName = document.DetectedLanguages[0].Iso6391Name;
KeyPhraseBatchResult phraseResult = client.KeyPhrases(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput(isoLanguageName, "1", textToInspect),
}));
var phrasesFound = phraseResult.Documents.FirstOrDefault();
if (phrasesFound == null)
{
throw new Exception("Failed processing message - no phrase result");
}
sb.AppendLine($"Document ID: {phrasesFound.Id} ");
sb.AppendLine("\t Key phrases:");
foreach (string keyphrase in phrasesFound.KeyPhrases)
{
sb.AppendLine("\t\t" + keyphrase);
var entitySearchApi = new EntitySearchAPI(new ApiKeyServiceClientCredentials(config["EntityKey"]));
var entityData = entitySearchApi.Entities.Search(keyphrase);
if (entityData?.Entities?.Value?.Count > 0)
{
// find the entity that represents the dominant one
var mainEntity = entityData.Entities.Value.Where(thing => thing.EntityPresentationInfo.EntityScenario == EntityScenario.DominantEntity).FirstOrDefault();
if (mainEntity != null)
{
sb.AppendLine($"Searched for {keyphrase} and found a dominant entity with this description:");
sb.AppendLine(mainEntity.Description);
}
else
{
sb.AppendLine($"Couldn't find a main entity for {keyphrase}");
}
}
else
{
sb.AppendLine($"No data returned for entity {keyphrase}");
}
}
// Extracting sentiment
sb.AppendLine("\n\n===== SENTIMENT ANALYSIS ======");
SentimentBatchResult sentimentResult = client.Sentiment(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput(isoLanguageName, "0", textToInspect),
}));
var sentiment = sentimentResult.Documents.FirstOrDefault();
if (sentiment == null)
{
throw new Exception("Failed processing message - no sentiment result");
}
// Printing sentiment results
sb.AppendLine($"Document ID: {sentiment.Id} , Sentiment Score: {sentiment.Score}");
}
return(sb.ToString());
}
public async Task <TextAnalyticsDocumentSentimentModel> CallTextAnalytics(string utterance)
{
// Create a client.
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Westeurope;
client.SubscriptionKey = _textAnalyticsSubscriptionKey;
var keysList = "";
var language = client.DetectLanguage(
new BatchInput(
new List <Input>()
{
new Input("0", utterance)
}));
foreach (var document in language.Documents)
{
Console.WriteLine("Document ID: {0} , Language: {1}", document.Id, document.DetectedLanguages[0].Name);
}
var lang = language.Documents.FirstOrDefault()?.DetectedLanguages.FirstOrDefault()?.Iso6391Name;
var keys = client.KeyPhrases(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput(lang, "0", utterance),
}));
var sentiment = client.Sentiment(new MultiLanguageBatchInput(new List <MultiLanguageInput>()
{
new MultiLanguageInput(lang, "0", utterance)
}));
//Si les sentiments sont nulls, on renvoie un objet vide
if (sentiment.Documents == null)
{
return(new TextAnalyticsDocumentSentimentModel());
}
{
var document = new TextAnalyticsDocumentSentimentModel
{
Text = utterance,
Score = sentiment.Documents.FirstOrDefault(x => x.Id == "0")?.Score,
Id = sentiment.Documents.FirstOrDefault()?.Id,
Language = lang
};
if (keys.Documents != null)
{
foreach (var item in keys.Documents.SelectMany(x => x.KeyPhrases).ToList())
{
document.KeyWords += item;
}
}
if (language.Documents == null)
{
return(document);
}
{
foreach (var item in language.Documents.SelectMany(x => x.DetectedLanguages).ToList())
{
document.DetectedLanguage += item.Name;
}
}
return(document);
}
}
static void Main(string[] args)
{
// Create a client.
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Eastus2;
client.SubscriptionKey = args[0];
Console.OutputEncoding = System.Text.Encoding.UTF8;
string[] fileEntries = Directory.GetFiles(args[1]);
foreach (string fileName in fileEntries)
{
FileStream fileStream = new FileStream(fileName, FileMode.Open);
FileStream out1 = new FileStream(fileName + ".KP.TXT", FileMode.Create);
FileStream out2 = new FileStream(fileName + ".SENT.TXT", FileMode.Create);
try
{ // Open the text file using a stream reader.
using (StreamReader sr = new StreamReader(fileStream))
{
String fullText = "";
// Read the stream to a string, and write the string to the console.
while (!sr.EndOfStream)
{
String line = sr.ReadLine();
if (!line.StartsWith("NOTE Confidence:") && !line.StartsWith("00:") && !line.StartsWith("WEBVTT"))
{
fullText = fullText + line + System.Environment.NewLine;
}
}
//Console.Write(fullText);
Console.WriteLine("\n\n===== KEY-PHRASE EXTRACTION ======");
KeyPhraseBatchResult result2 = client.KeyPhrases(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput("", "1", fullText)
}));
using (StreamWriter sw1 = new StreamWriter(out1))
{
// Printing keyphrases
foreach (var document in result2.Documents)
{
foreach (string keyphrase in document.KeyPhrases)
{
sw1.WriteLine(keyphrase);
}
/*
* Console.WriteLine("Document ID: {0} ", document.Id);
* Console.WriteLine("\t Key phrases:");
* foreach (string keyphrase in document.KeyPhrases)
* {
* Console.WriteLine("\t\t" + keyphrase);
* }
*/
}
}
// Extracting sentiment
Console.WriteLine("\n\n===== SENTIMENT ANALYSIS ======");
SentimentBatchResult result3 = client.Sentiment(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput("", "1", fullText)
}));
using (StreamWriter sw2 = new StreamWriter(out2))
{
// Printing keyphrases
foreach (var document in result3.Documents)
{
sw2.WriteLine("Sentiment Score: {0:0.00}", document.Score);
}
}
// Printing sentiment results
//foreach (var document in result3.Documents)
//{
//Console.WriteLine("Document ID: {0} , Sentiment Score: {1:0.00}", document.Id, document.Score);
//}
}
}
catch (Exception e)
{
Console.WriteLine("The file could not be read:");
Console.WriteLine(e.Message);
}
}
}
private void button_OK_Click(object sender, RoutedEventArgs e)
{
var data = new MultiLanguageBatchInput();
// Extracting text
var di = new DirectoryInfo(txtBox_pathToFiles.Text);
int index = 0;
var RegEx_SentenceDelimiter = new Regex(@"(\.|\!|\?)");
string fulltext = "";
foreach (FileInfo fi in di.GetFiles())
{
string path = fi.FullName;
string title = fi.Name;
var extractor = new TikaOnDotNet.TextExtraction.TextExtractor();
var extractionResult = extractor.Extract(path);
string text = extractionResult.Text;
text = Regex.Replace(text, @"[\r\n\t\f\v]", " ");
text = Regex.Replace(text, @"[^a-z.,!?]", " ", RegexOptions.IgnoreCase);
text = Regex.Replace(text, @"( +)", " ");
var values = new JObject();
JArray documents = new JArray();
Topic topic = new Topic();
int sentenceCount = RegEx_SentenceDelimiter.Split(text).Length;
//int factor = 1;
//if ((double) sentenceCount / 1000 <= 1) factor = 1;
//else factor = (sentenceCount / 1000) + 1;
List <string> sentences = new List <string>();
//if (useWastefulLogic)
//{
// if (sentenceCount < 100)
// {
// var splitFactor = (100 / sentenceCount) + 1;
// // splitFactor tells us, into how many pieces each sentence needs to be split
// foreach (var sentenceCandidate in RegEx_SentenceDelimiter.Split(text))
// {
// sentences.Add(sentenceCandidate);
// for (int j = 1; j <= splitFactor; j++)
// {
// sentences.Add(" ");
// }
// }
// }
// else if (100 < sentenceCount && sentenceCount < 1000)
// {
sentences = RegEx_SentenceDelimiter.Split(text).ToList();
// }
// else // sentenceCount >= 1000
// {
// int counter = 1;
// string t = "";
// int docId = 1;
// sentences = RegEx_SentenceDelimiter.Split(text).ToList();
// foreach (string sentence in sentences)
// {
// if (counter <= factor)
// {
// t += sentence;
// counter++;
// }
// else
// {
// Document d = new Document();
// d.id = docId;
// d.text = t;
// topic.documents.Add(d);
// t = "";
// t += sentence;
// counter = 1;
// docId++;
// }
// }
// }
//}
//else
//{
// sentences = RegEx_SentenceDelimiter.Split(text).ToList();
// int maxSentencesPerDocument = sentences.Count / 100;
// int counter = 1;
// string t = "";
// int docId = 1;
// foreach (string sentence in sentences)
// {
// if ((t + ". " + sentence).Length > maxSentenceLength || counter >= maxSentencesPerDocument)
// {
// Document d = new Document();
// d.id = docId;
// d.text = t;
// topic.documents.Add(d);
// t = "";
// t += sentence;
// counter = 1;
// docId++;
// }
// else
// {
// t += ". " + sentence;
// counter++;
// }
// }
//}
List <string> finalizedSentences = new List <string>();
string sentenceCandidate = "";
foreach (var sentence in sentences)
{
// sanitize
if (sentence.Length < 5)
{
continue;
}
if (sentenceCandidate.Length + sentence.Length > 5120)
{
finalizedSentences.Add(sentenceCandidate);
sentenceCandidate = sentence;
}
else
{
sentenceCandidate += " " + sentence;
}
}
var analyzable = new List <MultiLanguageInput>();
int i = 0;
foreach (var s in finalizedSentences)
{
if (s.Length > 10)
{
analyzable.Add(new MultiLanguageInput("en", i + "", s));
}
i++;
}
//analyzable.Add(new MultiLanguageInput("en", 0 + "", fulltext));
data.Documents = analyzable;
//topic.stopWords.Add("world");
//topic.stopPhrases.Add("world");
//string result = "";
ITextAnalyticsAPI client = new TextAnalyticsAPI();
client.AzureRegion = AzureRegions.Westus;
client.SubscriptionKey = key1;
//JsonSerializerSettings jss = new JsonSerializerSettings();
//jss.Formatting = Formatting.None;
//string json = values.ToString();
//json = JsonConvert.SerializeObject(topic, jss);
try
{
var result = client.KeyPhrases(data);
foreach (var row in result.Documents)
{
foreach (var kp in row.KeyPhrases)
{
AddMessage(kp);
}
}
//result = client.UploadString(url_topics, "POST", json);
//string requestId = client.ResponseHeaders.Get("operation-location");
//int topicCount = int.Parse(ddl_ResultsCount.SelectedItem.ToString());
//Thread thread = new Thread(delegate ()
//{
// GetDataAndUpdate(requestId, title, index, topicCount);
// // rest omitted for clarity
//});
//thread.IsBackground = true;
//thread.Start();
//// pause the main thread for a while to stop from getting 429 errors
//Thread.Sleep(60000);
}
catch (Exception ex)
{
AddMessage(title + ": " + ex.Message);
}
}
index++;
}
private async Task MessageReceivedAsync(IDialogContext context, IAwaitable <object> result)
{
var activity = await result as Activity;
var userText = activity.Text;
if (userText == null)
{
await context.PostAsync($"こんにちは！ルー語BOTだよ。ルー語にしたい文章を入れてみてね。");
}
else
{
// API Client の作成
var textAnalyticsClient = new TextAnalyticsAPI();
textAnalyticsClient.AzureRegion = textAnalyticsRegion;
textAnalyticsClient.SubscriptionKey = textAnalyticsSubKey;
var translatorClient = new TranslatorServiceClient(translatorSubKey);
var keyPhraseJa = new List <string>();
var keyPhraseEn = new List <string>();
// Text Analytics API を利用したキーワードの取得
try
{
var textAnalyticsResult = textAnalyticsClient.KeyPhrases(
new MultiLanguageBatchInput(
new List <MultiLanguageInput>()
{
new MultiLanguageInput("ja", "1", userText)
}));
foreach (var keyPhrase in textAnalyticsResult.Documents[0].KeyPhrases)
{
//await context.PostAsync($"Keyword= " + keyPhrase);
keyPhraseJa.Add(keyPhrase);
}
}
catch
{
await context.PostAsync($"Error: TextAnalytics API");
}
// Translator API を利用したキーワードの翻訳(日本語→英語)
try
{
foreach (var keyPhrase in keyPhraseJa)
{
var res = await translatorClient.TranslateAsync(keyPhrase, to : "en-us");
//await context.PostAsync($"Keyword in English = " + res);
keyPhraseEn.Add(res);
}
}
catch
{
await context.PostAsync($"Error: Translator API");
}
// キーワード(日本語→英語)の入れ替えと返答
for (int i = 0; i < keyPhraseJa.Count; i++)
{
userText = userText.Replace(keyPhraseJa[i], keyPhraseEn[i]);
}
await context.PostAsync(userText);
context.Wait(MessageReceivedAsync);
}
}
