public void ProcessExecute()
{
Themes.Clear();
int nRet = Engine.PrepareText(TextToAnalyse);
if (nRet == 0)
{
// Get themes
List <SalienceTheme> myThemes = Engine.GetDocumentThemes(String.Empty);
foreach (SalienceTheme aTheme in myThemes)
{
Themes.Add(new Theme(aTheme.sNormalizedTheme, aTheme.fScore, aTheme.nThemeType, aTheme.fSentiment, aTheme.nEvidence));
}
// Get named entities
List <SalienceEntity> myEntities = Engine.GetNamedEntities(String.Empty);
foreach (SalienceEntity anEntity in myEntities)
{
NamedEntities.Add(new NamedEntity(anEntity.sNormalizedForm, anEntity.sType, anEntity.fSentimentScore, anEntity.nEvidence, anEntity.nCount));
}
// Get summary
SalienceSummary mySummary = Engine.GetSummary(3, String.Empty);
Summary.SummaryText = mySummary.sSummary;
}
else
{
// there was an error, in which case this needs to be handled somehow
}
}
public void CanNotBeAttribute(string word, NamedEntities entities, bool expected)
{
var wordItem = ActualWordsHandler.InstanceSimple.WordFactory.CreateWord(word, "NN");
wordItem.Entity = entities;
var result = wordItem.CanNotBeAttribute();
Assert.AreEqual(expected, result);
}
private static void SortEntities(NamedEntities entities, List <NamedEntity> sortedEntities)
{
sortedEntities.Clear();
foreach (NamedEntity entity in entities)
{
sortedEntities.Add(entity);
}
sortedEntities.Sort(NamedEntitiesComparator);
}
// public methods
public void AddEntity(Entity e)
{
// add to entity list
Entities.Add(e);
// add to entity dictionary if its id is not empty
if (e.Id != null)
{
if (NamedEntities.ContainsKey(e.Id))
{
throw new ArgumentException();
}
NamedEntities.Add(e.Id, e);
}
}
virtual public void Update()
{
// pack all living entity to the front of entity list
// and remove unused named entities from dictionary
temporaryRemovedEntities.Clear();
int runningIndex = 0;
for (int i = 0; i < Entities.Count; i++)
{
Entity e = Entities[i];
if (e.CanRemove || (e.RemoveTime != null && Program.CurrentTime > e.RemoveTime))
{
if (e.Id != null)
{
NamedEntities.Remove(e.Id);
}
temporaryRemovedEntities.Add(e);
}
else
{
Entities[runningIndex++] = e;;
}
}
// resize the list to the size of living entities
if (runningIndex < Entities.Count)
{
Entities.RemoveRange(runningIndex, Entities.Count - runningIndex);
}
// raise OnRemove event from removed entities
foreach (var e in temporaryRemovedEntities)
{
e.RaiseOnRemove();
}
temporaryRemovedEntities.Clear();
// update all living entities
foreach (Entity e in Entities)
{
e.Update();
}
}
public TextAnalysisViewModel()
{
_pathInput = new PathInput {
LicensePath = Strings.Label_LicensePath, DataPath = Strings.Label_DataPath
};
_input = new Input {
InputText = Strings.Label_InputText_SampleString
};
_themes = new Themes();
_summary = new Summary();
_namedEntities = new NamedEntities();
_import = new Import();
_textToAnalyse = InputText;
try
{
Engine = new Salience(PathInput.LicensePath, PathInput.DataPath);
}
catch (SalienceException e)
{
//not sure how to handle this yet
}
}
public static int Main(string[] args)
{
if (args.Length < 1) {
Console.Error.WriteLine("Usage: RunMorphoCli ner_file");
return 1;
}
Console.Error.Write("Loading ner: ");
Ner ner = Ner.load(args[0]);
if (ner == null) {
Console.Error.WriteLine("Cannot load ner from file '{0}'", args[0]);
return 1;
}
Console.Error.WriteLine("done");
Forms forms = new Forms();
TokenRanges tokens = new TokenRanges();
NamedEntities entities = new NamedEntities();
List<NamedEntity> sortedEntities = new List<NamedEntity>();
Stack<int> openEntities = new Stack<int>();
Tokenizer tokenizer = ner.newTokenizer();
if (tokenizer == null) {
Console.Error.WriteLine("No tokenizer is defined for the supplied model!");
return 1;
}
XmlTextWriter xmlOut = new XmlTextWriter(Console.Out);
for (bool not_eof = true; not_eof; ) {
string line;
StringBuilder textBuilder = new StringBuilder();
// Read block
while ((not_eof = (line = Console.In.ReadLine()) != null) && line.Length > 0) {
textBuilder.Append(line).Append('\n');
}
if (not_eof) textBuilder.Append('\n');
// Tokenize and tag
string text = textBuilder.ToString();
tokenizer.setText(text);
int t = 0;
while (tokenizer.nextSentence(forms, tokens)) {
ner.recognize(forms, entities);
SortEntities(entities, sortedEntities);
for (int i = 0, e = 0; i < tokens.Count; i++) {
int token_start = (int)tokens[i].start, token_length = (int)tokens[i].length;
xmlOut.WriteString(text.Substring(t, token_start - t));
if (i == 0) xmlOut.WriteStartElement("sentence");
for (; e < sortedEntities.Count && sortedEntities[e].start == i; e++) {
xmlOut.WriteStartElement("ne");
xmlOut.WriteAttributeString("type", sortedEntities[e].type);
openEntities.Push((int)sortedEntities[e].start + (int)sortedEntities[e].length - 1);
}
xmlOut.WriteStartElement("token");
xmlOut.WriteString(text.Substring(token_start, token_length));
xmlOut.WriteEndElement();
for (; openEntities.Count > 0 && openEntities.Peek() == i; openEntities.Pop())
xmlOut.WriteEndElement();
if (i + 1 == tokens.Count) xmlOut.WriteEndElement();
t = token_start + token_length;
}
}
xmlOut.WriteString(text.Substring(t));
}
return 0;
}
private static void SortEntities(NamedEntities entities, List<NamedEntity> sortedEntities)
{
sortedEntities.Clear();
foreach (NamedEntity entity in entities)
sortedEntities.Add(entity);
sortedEntities.Sort(NamedEntitiesComparator);
}
public static int Main(string[] args)
{
if (args.Length < 1)
{
Console.Error.WriteLine("Usage: RunMorphoCli ner_file");
return(1);
}
Console.Error.Write("Loading ner: ");
Ner ner = Ner.load(args[0]);
if (ner == null)
{
Console.Error.WriteLine("Cannot load ner from file '{0}'", args[0]);
return(1);
}
Console.Error.WriteLine("done");
Forms forms = new Forms();
TokenRanges tokens = new TokenRanges();
NamedEntities entities = new NamedEntities();
List <NamedEntity> sortedEntities = new List <NamedEntity>();
Stack <int> openEntities = new Stack <int>();
Tokenizer tokenizer = ner.newTokenizer();
if (tokenizer == null)
{
Console.Error.WriteLine("No tokenizer is defined for the supplied model!");
return(1);
}
XmlTextWriter xmlOut = new XmlTextWriter(Console.Out);
for (bool not_eof = true; not_eof;)
{
string line;
StringBuilder textBuilder = new StringBuilder();
// Read block
while ((not_eof = (line = Console.In.ReadLine()) != null) && line.Length > 0)
{
textBuilder.Append(line).Append('\n');
}
if (not_eof)
{
textBuilder.Append('\n');
}
// Tokenize and tag
string text = textBuilder.ToString();
tokenizer.setText(text);
int t = 0;
while (tokenizer.nextSentence(forms, tokens))
{
ner.recognize(forms, entities);
SortEntities(entities, sortedEntities);
for (int i = 0, e = 0; i < tokens.Count; i++)
{
int token_start = (int)tokens[i].start, token_length = (int)tokens[i].length;
xmlOut.WriteString(text.Substring(t, token_start - t));
if (i == 0)
{
xmlOut.WriteStartElement("sentence");
}
for (; e < sortedEntities.Count && sortedEntities[e].start == i; e++)
{
xmlOut.WriteStartElement("ne");
xmlOut.WriteAttributeString("type", sortedEntities[e].type);
openEntities.Push((int)sortedEntities[e].start + (int)sortedEntities[e].length - 1);
}
xmlOut.WriteStartElement("token");
xmlOut.WriteString(text.Substring(token_start, token_length));
xmlOut.WriteEndElement();
for (; openEntities.Count > 0 && openEntities.Peek() == i; openEntities.Pop())
{
xmlOut.WriteEndElement();
}
if (i + 1 == tokens.Count)
{
xmlOut.WriteEndElement();
}
t = token_start + token_length;
}
}
xmlOut.WriteString(text.Substring(t));
}
return(0);
}
