public async Task NgramSentiment()
{
ActualWordsHandler.InstanceOpen.Container.Context.DisableFeatureSentiment = true;
var words = Path.Combine(TestContext.CurrentContext.TestDirectory, @"Adjustment/words.csv");
ISentimentDataHolder lexicon = SentimentDataHolder.Load(words);
var loader = SentimentDataHolder.Load(new[] { "veto it really" }.Select(item =>
new WordSentimentValueData(
item,
new SentimentValueData(2))));
lexicon.Merge(loader);
var text = "I Veto it really";
var result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest(text)).ConfigureAwait(false);
var document = result.Construct(ActualWordsHandler.InstanceOpen.WordFactory);
ActualWordsHandler.InstanceOpen.Container.Context.Lexicon = lexicon;
Text.Data.IParsedReview review = ActualWordsHandler.InstanceOpen.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create();
Assert.AreEqual(1, review.CalculateRawRating().StarsRating);
ActualWordsHandler.InstanceOpen.Container.Context.NGram = 3;
review = ActualWordsHandler.InstanceOpen.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create();
Assert.AreEqual(5, review.CalculateRawRating().StarsRating);
IRatingAdjustment adjustment = RatingAdjustment.Create(review, null);
var resultDocument = new DocumentFromReviewFactory(ActualWordsHandler.InstanceOpen.Container.Resolve <INRCDictionary>()).ReparseDocument(adjustment);
Assert.AreEqual(5, resultDocument.Stars);
Assert.AreEqual("I Veto it really", resultDocument.Text);
}
public async Task FullSentence()
{
var text = "This tale based on two Edgar Allen Poe pieces (\"The Fall of the House of Usher\", \"Dance of Death\" (poem) ) is actually quite creepy from beginning to end. It is similar to some of the old black-and-white movies about people that meet in an old decrepit house (for example, \"The Cat and the Canary\", \"The Old Dark House\", \"Night of Terror\" and so on). Boris Karloff plays a demented inventor of life-size dolls that terrorize the guests. He dies early in the film (or does he ? ) and the residents of the house are subjected to a number of terrifying experiences. I won't go into too much detail here, but it is definitely a must-see for fans of old dark house mysteries.<br /><br />Watch it with plenty of popcorn and soda in a darkened room.<br /><br />Dan Basinger 8/10";
ActualWordsHandler.InstanceSimple.Container.Context.DisableFeatureSentiment = true;
ActualWordsHandler.InstanceSimple.Container.Context.DisableInvertors = true;
string[] positiveAdj = { "good", "lovely", "excellent", "delightful", "perfect" };
string[] negativeAdj = { "bad", "horrible", "poor", "disgusting", "unhappy" };
var sentiment = new Dictionary <string, double>();
foreach (var item in positiveAdj)
{
sentiment[item] = 2;
}
foreach (var item in negativeAdj)
{
sentiment[item] = -2;
}
var adjustment = SentimentDataHolder.PopulateEmotionsData(sentiment);
var request = await textSplitter.Process(new ParseRequest(text)).ConfigureAwait(false);
var document = request.Construct(ActualWordsHandler.InstanceSimple.WordFactory);
ActualWordsHandler.InstanceSimple.Container.Context.Lexicon = adjustment;
var review = ActualWordsHandler.InstanceSimple.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create();
Assert.IsNull(review.CalculateRawRating().StarsRating);
SentimentValue[] sentiments = review.GetAllSentiments();
Assert.AreEqual(0, sentiments.Length);
}
public async Task TestReview(bool disableInvert)
{
var txt = "#paulryan #killed #rnc2016 #america #died #wisconsin no more EMOTICON_kissing_heart since you gave up on #trump, you don't represent #us";
var stream = new DictionaryStream(Path.Combine(path, "Library", "Standard", "EmotionLookupTable.txt"), new FileStreamSource());
var data = stream.ReadDataFromStream(double.Parse).ToDictionary(item => item.Word, item => item.Value, StringComparer.OrdinalIgnoreCase);
foreach (var item in data.Keys.ToArray().Where(k => !k.StartsWith("EMOTICON")))
{
data.Remove(item);
}
var lexicon = SentimentDataHolder.PopulateEmotionsData(data);
ActualWordsHandler.InstanceOpen.Container.Context.DisableInvertors = disableInvert;
var result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest(txt)).ConfigureAwait(false);
var document = result.Construct(ActualWordsHandler.InstanceOpen.WordFactory);
ActualWordsHandler.InstanceOpen.Container.Context.Lexicon = lexicon;
Text.Data.IParsedReview review = ActualWordsHandler.InstanceOpen.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create();
MachineLearning.Mathematics.RatingData ratings = review.CalculateRawRating();
Assert.AreEqual(1, review.Sentences.Count);
Assert.AreEqual(disableInvert, ratings.IsPositive);
}
public void Setup()
{
var path = ActualWordsHandler.InstanceSimple.Configuration.GetConfiguration("Resources");
path = Path.Combine(path, @"Library\Standard");
var stream = new DictionaryStream(Path.Combine(path, "EmotionLookupTable.txt"), new FileStreamSource());
var data = stream.ReadDataFromStream(double.Parse).ToDictionary(item => item.Word, item => item.Value, StringComparer.OrdinalIgnoreCase);
sentimentData = SentimentDataHolder.PopulateEmotionsData(data);
}
public void Setup()
{
var loader = new LexiconConfigLoader(ApplicationLogging.LoggerFactory.CreateLogger <LexiconConfigLoader>());
var config = loader.Load(TestContext.CurrentContext.TestDirectory);
var path = config.GetFullPath(item => item.Model);
var stream = new DictionaryStream(Path.Combine(path, "EmotionLookupTable.txt"), new FileStreamSource());
var data = stream.ReadDataFromStream(double.Parse).ToDictionary(item => item.Word, item => item.Value, StringComparer.OrdinalIgnoreCase);
sentimentData = SentimentDataHolder.PopulateEmotionsData(data);
}
public void Adjust(string word, double weight, double sentiment)
{
var table = new Dictionary <string, double>();
table[word] = weight;
sentimentData = SentimentDataHolder.PopulateEmotionsData(table);
Text.Words.IWordItem wordItem = ActualWordsHandler.InstanceSimple.WordFactory.CreateWord(word, "NN");
SentimentValue measurement = sentimentData.MeasureSentiment(wordItem);
Assert.AreEqual(sentiment, measurement.DataValue.Value);
}
public async Task Adjusted()
{
ActualWordsHandler.InstanceOpen.Container.Context.DisableFeatureSentiment = true;
var words = Path.Combine(TestContext.CurrentContext.TestDirectory, @"Adjustment/words.csv");
ISentimentDataHolder lexicon = SentimentDataHolder.Load(words);
var text = "I Veto it";
var result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest(text)).ConfigureAwait(false);
var document = result.Construct(ActualWordsHandler.InstanceOpen.WordFactory);
ActualWordsHandler.InstanceOpen.Container.Context.Lexicon = lexicon;
Text.Data.IParsedReview review = ActualWordsHandler.InstanceOpen.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create();
Assert.AreEqual(1, review.CalculateRawRating().StarsRating);
}
public async Task AdjustSentiment(string word, int value, double?rating)
{
var request = await textSplitter.Process(new ParseRequest("Like or hate it")).ConfigureAwait(false);
var document = request.Construct(ActualWordsHandler.InstanceSimple.WordFactory);
var review = ActualWordsHandler.InstanceSimple.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create();
var adjustment = new LexiconRatingAdjustment(
review,
SentimentDataHolder.Load(new[]
{
new WordSentimentValueData(word, new SentimentValueData(value)),
}));
adjustment.CalculateRating();
Assert.AreEqual(rating, adjustment.Rating.RawRating);
}
public async Task TestCustom(string text, bool useFallback, double?rating)
{
var sentiment = new Dictionary <string, double>();
sentiment["hate"] = 2;
var adjustment = SentimentDataHolder.PopulateEmotionsData(sentiment);
ActualWordsHandler.InstanceSimple.Container.Context.UseBuiltInSentiment = useFallback;
var request = await textSplitter.Process(new ParseRequest(text)).ConfigureAwait(false);
var document = request.Construct(ActualWordsHandler.InstanceSimple.WordFactory);
ActualWordsHandler.InstanceSimple.Container.Context.Lexicon = adjustment;
var review = ActualWordsHandler.InstanceSimple.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create();
Assert.AreEqual(rating, review.CalculateRawRating().StarsRating);
}
protected override Task Execute(CancellationToken token)
{
Logger.LogInformation("Initialize...");
container.Context.DisableFeatureSentiment = Config.InvertOff;
Logger.LogInformation("Processing...");
ISentimentDataHolder sentimentAdjustment = default;
if (!string.IsNullOrEmpty(Config.Weights))
{
Logger.LogInformation("Adjusting Embeddings sentiments using [{0}] ...", Config.Weights);
sentimentAdjustment = SentimentDataHolder.Load(Config.Weights);
}
IObservable <IParsedDocumentHolder> review = GetAllDocuments();
return(Process(review.Select(SynchronizedReviews).Merge(), container, sentimentAdjustment));
}
public async Task SentimentTests(SentimentTestData data)
{
log.LogInformation("SentimentTests: {0}", data);
string file;
switch (data.Category)
{
case ProductCategory.Electronics:
file = "Electronics.csv";
break;
case ProductCategory.Video:
file = "video.csv";
break;
case ProductCategory.Kitchen:
file = "kitchen.csv";
break;
case ProductCategory.Medic:
case ProductCategory.Games:
case ProductCategory.Toys:
case ProductCategory.Book:
throw new NotImplementedException();
default:
throw new ArgumentOutOfRangeException();
}
ISentimentDataHolder holder = SentimentDataHolder.Load(Path.Combine(TestContext.CurrentContext.TestDirectory, "Sentiments", file));
var runner = new TestRunner(TestHelper.Instance, data);
Analysis.Processing.ITestingClient testing = runner.Active.GetTesting();
runner.Active.Context.Lexicon = holder;
testing.DisableAspects = true;
testing.DisableSvm = true;
testing.TrackArff = true;
testing.Init();
await testing.Process(runner.Load()).LastOrDefaultAsync();
testing.Save(Path.Combine(TestContext.CurrentContext.TestDirectory, "Word2Vec"));
Assert.AreEqual(data.Performance, testing.GetPerformanceDescription());
Assert.AreEqual(data.Errors, testing.Errors);
}
public async Task Process(IConnectionContext target, SentimentMessage message, CancellationToken token)
{
if (message == null)
{
throw new ArgumentNullException(nameof(message));
}
var request = message.Request;
if (request?.Documents == null)
{
throw new Exception("Nothing to process");
}
if (request.Documents.Length > 500)
{
throw new Exception("Too many documents. Maximum is 500");
}
var completed = new CompletedMessage();
try
{
var monitor = new PerformanceMonitor(request.Documents.Length);
using (Observable.Interval(TimeSpan.FromSeconds(10))
.Subscribe(item => logger.LogInformation(monitor.ToString())))
{
ISentimentDataHolder lexicon = default;
if (request.Dictionary != null &&
request.Dictionary.Count > 0)
{
logger.LogInformation("Creating custom dictionary with {0} words", request.Dictionary.Count);
lexicon = SentimentDataHolder.Load(request.Dictionary.Select(item =>
new WordSentimentValueData(
item.Key,
new SentimentValueData(item.Value))));
}
if ((lexicon == null || request.AdjustDomain) &&
!string.IsNullOrEmpty(request.Domain))
{
logger.LogInformation("Using Domain dictionary [{0}]", request.Domain);
var previous = lexicon;
lexicon = lexiconLoader.GetLexicon(request.Domain);
if (previous != null)
{
lexicon.Merge(previous);
}
}
string modelLocation = null;
if (!string.IsNullOrEmpty(request.Model))
{
logger.LogInformation("Using model path: {0}", request.Model);
modelLocation = storage.GetLocation(target.Connection.User, request.Model, ServiceConstants.Model);
if (!Directory.Exists(modelLocation))
{
throw new ApplicationException($"Can't find model {request.Model}");
}
}
using (var scope = provider.CreateScope())
{
var container = scope.ServiceProvider.GetService <ISessionContainer>();
container.Context.NGram = 3;
container.Context.ExtractAttributes = request.Emotions;
var client = container.GetTesting(modelLocation);
var converter = scope.ServiceProvider.GetService <IDocumentConverter>();
client.Init();
client.Pipeline.ResetMonitor();
if (lexicon != null)
{
client.Lexicon = lexicon;
}
await client.Process(request.Documents.Select(item => converter.Convert(item, request.CleanText))
.ToObservable())
.Select(item =>
{
monitor.Increment();
return(item);
})
.Buffer(TimeSpan.FromSeconds(5), 10, scheduler)
.Select(async item =>
{
var result = new ResultMessage <Document> {
Data = item.Select(x => x.Processed).ToArray()
};
await target.Write(result, token).ConfigureAwait(false);
return(Unit.Default);
})
.Merge();
}
logger.LogInformation("Completed with final performance: {0}", monitor);
completed.Message = "Testing Completed";
await target.Write(completed, token).ConfigureAwait(false);
}
}
catch (Exception e)
{
completed.Message = e.Message;
await target.Write(completed, token).ConfigureAwait(false);
completed.IsError = true;
throw;
}
}