private WordVectorsData GetVector(WordEx word)
{
WordVectorsData vectors = wordVectors.GetSafeCreate(word.Text, () => new WordVectorsData(word));
vectors.CreateNewVector();
return(vectors);
}
public static Document Construct(this LightDocument document, IWordFactory factory)
{
var result = new Document(document.Text);
result.Author = document.Author;
result.DocumentTime = document.DocumentTime;
result.Id = document.Id;
document.Title = document.Title;
foreach (var sentence in document.Sentences)
{
var resultSentence = new SentenceItem(sentence.Text);
result.Add(resultSentence);
if (sentence.Words != null)
{
for (var i = 0; i < sentence.Words.Length; i++)
{
var word = sentence.Words[i];
var wordItem = factory.CreateWord(word.Text, word.Tag);
wordItem.WordIndex = i;
WordEx wordData = WordExFactory.Construct(wordItem);
wordData.Phrase = word.Phrase;
resultSentence.Add(wordData);
}
}
}
return(result);
}
public NRCRecord FindRecord(WordEx word)
{
if (!loaded)
{
throw new InvalidOperationException("Not loaded");
}
NRCRecord nrcRecord = null;
foreach (var text in word.GetPossibleText())
{
nrcRecord = FindRecord(text);
if (nrcRecord != null)
{
break;
}
}
if (nrcRecord == null)
{
return(null);
}
nrcRecord = (NRCRecord)nrcRecord.Clone();
if (word.IsInverted)
{
nrcRecord.Invert();
}
return(nrcRecord);
}
private void PopulateAttributes(SentimentVector vector, WordEx word, IWordItem original)
{
var record = nrcDictionary.FindRecord(word);
vector.ExtractData(record);
word.Emotions = record?.GetDefinedCategories().ToArray();
word.Attributes = original.Inquirer.Records.SelectMany(item => item.Description.Attributes).ToArray();
}
public static bool IsPronoun(this WordEx word)
{
if (word is null)
{
throw new ArgumentNullException(nameof(word));
}
return(word.Tag.WordType == WordType.Pronoun || WordTypeResolver.Instance.IsPronoun(word.Text));
}
public WordStyle(WordEx word)
{
if (word is null)
{
throw new ArgumentNullException(nameof(word));
}
Word = word.Text;
}
public static int CountSyllables(this WordEx word)
{
if (word is null)
{
throw new ArgumentNullException(nameof(word));
}
return(EnglishSyllableCounter.Instance.CountSyllables(word.Text));
}
public static bool IsCoordinatingConjunction(this WordEx word)
{
if (word is null)
{
throw new ArgumentNullException(nameof(word));
}
return(word.Tag == POSTags.Instance.CC || WordTypeResolver.Instance.IsCoordinatingConjunctions(word.Text));
}
public void CountSyllables()
{
WordEx word = new WordEx("creatures");
Assert.AreEqual(2, word.CountSyllables());
word = new WordEx("creature");
Assert.AreEqual(2, word.CountSyllables());
}
public void AddContext()
{
var addingWord = new WordEx(new SimpleWord("Test"));
var vectorData = new WordsContext(word);
vectorData.AddContext(addingWord);
Assert.AreEqual(1, vectorData.Words.Count);
vectorData.AddContext(addingWord);
Assert.AreEqual(2, vectorData.Words.Count);
}
public InquirerData GetData(WordEx word)
{
wordLevelFingerPrint.TryGetValue(word, out Dictionary <string, bool> data);
return(data == null
? null
: new InquirerData
{
Categories = data.Where(item => item.Value).Select(item => item.Key).ToArray()
});
}
public static bool IsDigit(this WordEx word)
{
if (word is null)
{
throw new ArgumentNullException(nameof(word));
}
return(word.EntityType == NamedEntities.Number ||
word.EntityType == NamedEntities.Percent ||
word.EntityType == NamedEntities.Money ||
word.EntityType == NamedEntities.Ordinal ||
word.Tag == POSTags.Instance.CD);
}
public static bool IsWordType(this IPOSTagger tagger, WordEx word, WordType type)
{
if (tagger is null)
{
throw new ArgumentNullException(nameof(tagger));
}
if (word is null)
{
throw new ArgumentNullException(nameof(word));
}
return(word.Tag.WordType == type || tagger.GetTag(word.Text).WordType == type);
}
public IEnumerable <IPhrase> GetPhrases(IWordItem word)
{
if (word == null)
{
throw new ArgumentNullException(nameof(word));
}
log.LogDebug("GetPhrases {0}", word);
IWordItem[] currentWords = word.Relationship.Part.Occurrences
.Where(item => !item.CanNotBeFeature() && !item.IsSentiment).ToArray();
if (currentWords.Length <= 1)
{
yield break;
}
var all = string.Join(" ", currentWords.Select(item => item.Text).ToArray());
var wordIndex = Array.IndexOf(currentWords, word);
if (wordIndex < 0)
{
log.LogDebug("{0} is not found in important list in <{1}>", word, all);
yield break;
}
var nGramBlocks = new List <NGramBlock>();
var wordsTable = new Dictionary <WordEx, IWordItem>();
var words = new WordEx[currentWords.Length];
foreach (IWordItem item in currentWords)
{
WordEx wordEx = WordExFactory.Construct(item);
words[wordsTable.Count] = wordEx;
wordsTable[wordEx] = item;
}
nGramBlocks.AddRange(words.GetNearNGram(wordIndex, 3));
nGramBlocks.AddRange(words.GetNearNGram(wordIndex, 2));
foreach (NGramBlock nGramBlock in nGramBlocks)
{
IPhrase phrase = handler.CreatePhrase("NP");
foreach (WordEx occurence in nGramBlock.WordOccurrences)
{
phrase.Add(wordsTable[occurence]);
}
yield return(phrase);
}
}
public WordEx Construct(string word)
{
if (string.IsNullOrEmpty(word))
{
throw new ArgumentException("Value cannot be null or empty.", nameof(word));
}
var wordEx = new WordEx(word);
wordEx.Type = tagger.GetTag(word).Tag;
wordEx.IsStop = Words.WordTypeResolver.Instance.IsStop(word);
wordEx.IsInvertor = Words.WordTypeResolver.Instance.IsInvertor(word);
wordEx.Raw = raw.GetWord(word);
return(wordEx);
}
public void AddContext(WordEx word)
{
if (word == null)
{
throw new ArgumentNullException(nameof(word));
}
if (string.Compare(word.Text, Word.Text, StringComparison.OrdinalIgnoreCase) == 0)
{
log.LogDebug("This is owner word: {0}", word.Text);
return;
}
Words.Add(word);
}
public void Clone()
{
item.Add("One");
item.Add("Two");
var word = new WordEx("T");
word.CalculatedValue = 2;
item.Add(word);
var sentence = (SentenceItem)item.Clone();
Assert.AreEqual(3, sentence.Words.Count);
Assert.AreEqual(item.CalculateSentiment().RawRating, sentence.CalculateSentiment().RawRating);
Assert.AreEqual("One", sentence.Words[0].Text);
Assert.AreEqual("Two", sentence.Words[1].Text);
Assert.AreEqual("T", sentence.Words[2].Text);
}
public void Serialize()
{
WordEx word = new WordEx(new SimpleWord("Test"));
word.Value = 11.11;
var json = JsonConvert.SerializeObject(word);
WordEx deserialized = JsonConvert.DeserializeObject <WordEx>(json);
Assert.AreEqual(word.UnderlyingWord.Text, deserialized.UnderlyingWord.Text);
Assert.AreEqual(11.11, deserialized.Value);
XDocument doc = word.XmlSerialize();
deserialized = doc.XmlDeserialize <WordEx>();
Assert.AreEqual(11.11, deserialized.Value);
}
public static Document Construct(this LightDocument document, IWordFactory factory)
{
var result = new Document(document.Text);
result.Author = document.Author;
result.DocumentTime = document.DocumentTime;
result.Id = document.Id;
document.Title = document.Title;
foreach (var sentence in document.Sentences)
{
var resultSentence = new SentenceItem(sentence.Text);
result.Add(resultSentence, false);
if (sentence.Words != null)
{
for (var i = 0; i < sentence.Words.Length; i++)
{
var word = sentence.Words[i];
var wordItem = factory.CreateWord(word.Text, word.Tag);
wordItem.WordIndex = i;
WordEx wordData = WordExFactory.Construct(wordItem);
wordData.Phrase = word.Phrase;
if (!string.IsNullOrEmpty(word.Entity))
{
if (entityCache.TryGetValue(word.Entity, out var entity))
{
wordData.EntityType = entity;
}
else
{
wordData.CustomEntity = word.Entity;
}
}
else
{
wordData.EntityType = NamedEntities.None;
}
resultSentence.Add(wordData);
}
}
}
return(result);
}
public void Clone()
{
WordEx word = new WordEx("Test");
word.Value = 11.11;
word.CalculatedValue = 2;
word.Id = 1;
word.IsInvertor = true;
word.EntityType = NamedEntities.Date;
var wordClone = (WordEx)word.Clone();
Assert.AreEqual(11.11, wordClone.Value);
Assert.AreEqual(2, wordClone.CalculatedValue);
Assert.AreEqual(true, wordClone.IsInvertor);
Assert.AreEqual(1, wordClone.Id);
Assert.AreEqual(NamedEntities.Date, wordClone.EntityType);
}
public static bool IsWordType(this IPOSTagger tagger, WordEx word, BasePOSType posType)
{
if (tagger is null)
{
throw new ArgumentNullException(nameof(tagger));
}
if (word is null)
{
throw new ArgumentNullException(nameof(word));
}
if (posType is null)
{
throw new ArgumentNullException(nameof(posType));
}
return(word.Tag == posType || tagger.GetTag(word.Text) == posType);
}
public static WordEx Construct(IWordItem item)
{
if (item == null)
{
throw new System.ArgumentNullException(nameof(item));
}
var word = new WordEx(item);
word.NormalizedEntity = item.NormalizedEntity;
word.EntityType = item.Entity;
word.Type = item.POS.Tag;
word.IsAspect = item.IsFeature;
word.IsStop = item.IsStopWord;
word.IsInvertor = item.IsInvertor;
word.IsInverted = item.Relationship?.Inverted != null;
word.Phrase = item.Parent?.Text;
word.Raw = item.Stemmed;
return(word);
}
public WordVectorsData(WordEx word)
{
Word = word ?? throw new ArgumentNullException(nameof(word));
Vectors = new List <WordsContext>();
}
public WordsContext(WordEx word)
{
Word = word ?? throw new ArgumentNullException(nameof(word));
Words = new List <WordEx>();
}
