public static string GenerateMask(this IWordItem wordItem, bool pure)
{
var word = wordItem.Stemmed;
if (!word.HasLetters())
{
return(string.Empty);
}
if (wordItem.IsFeature)
{
if (pure)
{
return(word);
}
word = "xxxFeature";
}
if (wordItem.IsInvertor)
{
return(!wordItem.IsAttached() ? "xxxNOTxxx" : null);
}
if (wordItem.Relationship.Inverted != null)
{
return(GetInvertedMask(word));
}
return(wordItem.IsFeature ? null : word);
}
public SentimentValue CheckSentiment(IWordItem word)
{
if (Context.DisableFeatureSentiment &&
word.IsFeature)
{
return(null);
}
if (Context.Lexicon != null)
{
SentimentValue sentiment = Context.Lexicon.MeasureSentiment(word);
if (sentiment != null ||
!Context.UseBuiltInSentiment)
{
return(sentiment);
}
var builtIn = inner.CheckSentiment(word);
if (builtIn != null)
{
// original max strength is 3 we move it save scale as custom and also make it 4 time less powerfull
return(new SentimentValue(word, builtIn.Span, builtIn.DataValue.Value / 3 * Context.Lexicon.AverageStrength / 4));
}
else
{
return(null);
}
}
return(inner.CheckSentiment(word));
}
private int GetDistance(IWordItem sentiment)
{
var parentIndex = parent.Owner.WordIndex;
var distance = Math.Abs(sentiment.WordIndex - parentIndex);
foreach (var relatedQuant in sentiment.Relationship.PriorQuants)
{
var currentDistance = Math.Abs(relatedQuant.WordIndex - parentIndex);
if (currentDistance < distance)
{
distance = currentDistance;
}
}
if (sentiment.Relationship.Inverted != null)
{
var currentDistance = Math.Abs(sentiment.Relationship.Inverted.WordIndex - parentIndex);
if (currentDistance < distance)
{
distance = currentDistance;
}
}
return(distance);
}
public bool IsInvertAdverb(IWordItem word)
{
if (negating.ContainsKey(word.Text))
{
if (negatingRule.TryGetValue(WordItemType.Invertor, out WordRepairRule rule))
{
var value = new WordRepairRuleEngine(word, rule).Evaluate();
if (value.HasValue)
{
return(value.Value);
}
}
return(true);
}
var evaluationValue = new WordRepairRuleEngine(word, FindRepairRule(word)).Evaluate();
if (evaluationValue.HasValue)
{
return(evaluationValue.Value);
}
return(false);
}
public static bool TryGetWordValue <T>(this IDictionary <string, T> table, IWordItem word, out T value)
{
if (table == null)
{
throw new System.ArgumentNullException(nameof(table));
}
if (word == null)
{
throw new System.ArgumentNullException(nameof(word));
}
value = default;
if (table.Count == 0)
{
return(false);
}
foreach (var text in word.Relationship.Views)
{
if (table.TryGetValue(text, out value))
{
return(true);
}
}
return(false);
}
public static NRCRecord FindRecord(this INRCDictionary dictionary, IWordItem word)
{
NRCRecord nrcRecord = null;
foreach (var text in word.GetPossibleText())
{
nrcRecord = dictionary.FindRecord(text);
if (nrcRecord != null)
{
break;
}
}
if (nrcRecord == null)
{
return(null);
}
nrcRecord = (NRCRecord)nrcRecord.Clone();
if (word.Relationship?.Inverted != null)
{
nrcRecord.Invert();
}
return(nrcRecord);
}
public void AddItem(IWordItem item)
{
if (item == null)
{
return;
}
if (!item.IsSimple)
{
throw new ProcessingException("Can add only simple word item type");
}
if (item.IsSimpleConjunction())
{
return;
}
if (items.Count > 0)
{
var previous = items[items.Count - 1];
item.Relationship.Previous = previous;
previous.Relationship.Next = item;
}
items.Add(item);
if (item.Parent != null)
{
item.Parent.Relationship.Part = this;
}
item.Relationship.Part = this;
AddRelationship(item);
windowItems.Add(item, item);
windowItems.Increment();
}
public static bool TryGetWordValue <T>(this MaskDictionary <T> table, IWordItem word, out T value)
{
if (table == null)
{
throw new System.ArgumentNullException(nameof(table));
}
if (word == null)
{
throw new System.ArgumentNullException(nameof(word));
}
if (table.TryGetValue(word.Text, out value))
{
return(true);
}
if (!string.IsNullOrEmpty(word.Stemmed) &&
word.Stemmed != word.Text)
{
if (table.TryGetValue(word.Stemmed, out value))
{
return(true);
}
}
return(false);
}
private void PopulateAttributes(SentimentVector vector, WordEx word, IWordItem original)
{
var record = nrcDictionary.FindRecord(word);
vector.ExtractData(record);
word.Emotions = record?.GetDefinedCategories().ToArray();
word.Attributes = original.Inquirer.Records.SelectMany(item => item.Description.Attributes).ToArray();
}
public static bool IsConjunction(this IWordItem word)
{
return(word.POS.WordType == WordType.Conjunction ||
WordTypeResolver.Instance.IsInvertingConjunction(word.Text) ||
WordTypeResolver.Instance.IsSpecialEndSymbol(word.Text) ||
WordTypeResolver.Instance.IsRegularConjunction(word.Text) ||
WordTypeResolver.Instance.IsSubordinateConjunction(word.Text));
}
private void AddRelationship(IWordItem currentItem)
{
foreach (var item in windowItems.Keys)
{
currentItem.Relationship.Add(item);
item.Relationship.Add(currentItem);
}
}
public static bool IsKnown(this IContextWordsHandler instance, IWordItem word)
{
return(instance.IsQuestion(word) ||
instance.IsFeature(word) ||
instance.IsInvertAdverb(word) ||
instance.IsSentiment(word) ||
instance.MeasureQuantifier(word) > 0);
}
public void Remove(IWordItem feature)
{
if (feature is null)
{
throw new ArgumentNullException(nameof(feature));
}
aspectsTable.Remove(feature.Text);
}
public void AddFeature(IWordItem feature)
{
if (feature is null)
{
throw new ArgumentNullException(nameof(feature));
}
aspectsTable[feature.Text] = feature;
}
public bool IsAttribute(IWordItem word)
{
if (word is null)
{
throw new ArgumentNullException(nameof(word));
}
return(attributesTable.TryGetWordValue(word, out _));
}
private WordRepairRule FindRepairRule(IWordItem word)
{
if (!word.IsSimple)
{
return(null);
}
return(negatingRepairRule.TryGetWordValue(word, out WordRepairRule rule) ? rule : null);
}
public bool IsInvertAdverb(IWordItem word)
{
if (Context.DisableInvertors)
{
return(false);
}
return(inner.IsInvertAdverb(word));
}
public void Add(IWordItem relatedWord)
{
if (relatedWord == null)
{
throw new System.ArgumentNullException(nameof(relatedWord));
}
related.Add(relatedWord);
Reset();
}
public void AddWord(IWordItem wordItem)
{
if (wordItem == null)
{
throw new ArgumentNullException(nameof(wordItem));
}
Interlocked.Increment(ref total);
table[wordItem.Text] = wordItem;
}
public SentimentCalculator(SentimentValue value)
{
if (value is null)
{
throw new ArgumentNullException(nameof(value));
}
wordItem = value.Owner;
sentimentValue = value;
}
public IParsedReview Create()
{
if (review != null)
{
return(review);
}
review = new ParsedReview(nrcDictionary, document, manager.Context);
foreach (var sentence in document.Sentences)
{
CreateSentence(sentence);
IPhrase phrase = null;
string phraseWord = null;
for (var i = 0; i < sentence.Words.Count; i++)
{
var documentWord = sentence.Words[i];
if (documentWord.Phrase != null)
{
if (phraseWord != documentWord.Phrase)
{
phraseWord = documentWord.Phrase;
phrase = documentWord.UnderlyingWord as IPhrase ??
wordsFactory.CreatePhrase(phraseWord);
}
}
else
{
phrase = null;
phraseWord = null;
}
// !! we need to create new - because if we use underlying
// we can lose if words is changed to aspect
IWordItem word = wordsFactory.CreateWord(documentWord.Text, documentWord.POS);
word.NormalizedEntity = documentWord.NormalizedEntity;
word.Entity = documentWord.EntityType;
word.CustomEntity = documentWord.CustomEntity;
word.WordIndex = i;
AddWord(word, i == sentence.Words.Count - 1);
phrase?.Add(word);
}
}
foreach (var sentence in review.Sentences)
{
foreach (var phrase in sentence.Occurrences.GetPhrases().Where(item => item.AllWords.Count() > 1))
{
phrase.IsSentiment = manager.IsSentiment(phrase);
phrase.IsFeature = manager.IsFeature(phrase);
phrase.IsTopAttribute = manager.IsAttribute(phrase);
}
}
return(review);
}
public bool IsStop(IWordItem wordItem)
{
if (wordItem.Text.Length <= 1 ||
stopWords.ContainsKey(wordItem.Text) ||
stopPos.ContainsKey(wordItem.POS.Tag))
{
return(true);
}
return(false);
}
protected void AddItem(IWordItem item, string word, double value)
{
if (string.IsNullOrEmpty(word))
{
throw new ArgumentException("message", nameof(word));
}
var addedCell = !table.TryGetValue(word, out var cell) ? new TextVectorCell(item, word, value) : new TextVectorCell(item, word, cell.Value + value);
table[word] = addedCell;
}
public SentimentValue MeasureSentiment(IWordItem word)
{
if (!EmotionsTable.TryGetWordValue(word, out var value))
{
return(MeasureLookupSentiment(word));
}
var sentiment = new SentimentValue(word, value.Word, new SentimentValueData(value.Data.Value));
return(sentiment);
}
private void AddItemCell(IWordItem wordItem)
{
var word = GetWord(wordItem);
if (string.IsNullOrEmpty(word))
{
return;
}
AddItem(wordItem, word, GetValue(wordItem));
}
public bool IsFeature(IWordItem word)
{
if (word.CanNotBeFeature())
{
return(false);
}
var value = Context.Aspect != null && Context.Aspect.IsAspect(word);
return(value);
}
public IEnumerable <IPhrase> GetPhrases(IWordItem word)
{
if (word == null)
{
throw new ArgumentNullException(nameof(word));
}
log.LogDebug("GetPhrases {0}", word);
IWordItem[] currentWords = word.Relationship.Part.Occurrences
.Where(item => !item.CanNotBeFeature() && !item.IsSentiment).ToArray();
if (currentWords.Length <= 1)
{
yield break;
}
var all = string.Join(" ", currentWords.Select(item => item.Text).ToArray());
var wordIndex = Array.IndexOf(currentWords, word);
if (wordIndex < 0)
{
log.LogDebug("{0} is not found in important list in <{1}>", word, all);
yield break;
}
var nGramBlocks = new List <NGramBlock>();
var wordsTable = new Dictionary <WordEx, IWordItem>();
var words = new WordEx[currentWords.Length];
foreach (IWordItem item in currentWords)
{
WordEx wordEx = WordExFactory.Construct(item);
words[wordsTable.Count] = wordEx;
wordsTable[wordEx] = item;
}
nGramBlocks.AddRange(words.GetNearNGram(wordIndex, 3));
nGramBlocks.AddRange(words.GetNearNGram(wordIndex, 2));
foreach (NGramBlock nGramBlock in nGramBlocks)
{
IPhrase phrase = handler.CreatePhrase("NP");
foreach (WordEx occurence in nGramBlock.WordOccurrences)
{
phrase.Add(wordsTable[occurence]);
}
yield return(phrase);
}
}
private string GetWord(IWordItem wordItem)
{
if (GenerateUsingImportantOnly &&
wordItem.POS.WordType != WordType.Adjective &&
wordItem.Entity != NamedEntities.Hashtag &&
!wordItem.IsTopAttribute &&
!wordItem.IsSentiment &&
!wordItem.IsFeature &&
!wordItem.IsInvertor &&
wordItem.Relationship?.Inverted == null)
{
return(null);
}
return(wordItem.GenerateMask(UsePureWord));
}
private SentimentValue MeasureLookupSentiment(IWordItem word)
{
if (word is IPhrase ||
!EmotionsLookup.TryGetWordValue(word, out var value))
{
return(null);
}
var isInverted = word.Text.EndsWith("free", StringComparison.OrdinalIgnoreCase) ||
word.Text.EndsWith("proof", StringComparison.OrdinalIgnoreCase) ||
word.Text.EndsWith("less", StringComparison.OrdinalIgnoreCase);
var invertingEnd = isInverted ? -1 : 1;
var sentiment = new SentimentValue(word, word.Text, new SentimentValueData(value.Value * invertingEnd));
return(sentiment);
}
public static bool IsNoise(this IWordItem word)
{
if (word.Inquirer?.Records.Any(
item =>
item.Description.Syntactic.Determiner.IsDeterminer ||
item.Description.Syntactic.Interrogative.IsInterrogatives ||
item.Description.Syntactic.Verb.IsHave ||
item.Description.Syntactic.Verb.IsModal ||
item.Description.Syntactic.Verb.IsDoVerb ||
item.Description.Syntactic.Verb.IsTo) == true)
{
return(true);
}
return(false);
}
