public async Task NgramSentiment() { ActualWordsHandler.InstanceOpen.Container.Context.DisableFeatureSentiment = true; var words = Path.Combine(TestContext.CurrentContext.TestDirectory, @"Adjustment/words.csv"); ISentimentDataHolder lexicon = SentimentDataHolder.Load(words); var loader = SentimentDataHolder.Load(new[] { "veto it really" }.Select(item => new WordSentimentValueData( item, new SentimentValueData(2)))); lexicon.Merge(loader); var text = "I Veto it really"; var result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest(text)).ConfigureAwait(false); var document = result.Construct(ActualWordsHandler.InstanceOpen.WordFactory); ActualWordsHandler.InstanceOpen.Container.Context.Lexicon = lexicon; Text.Data.IParsedReview review = ActualWordsHandler.InstanceOpen.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create(); Assert.AreEqual(1, review.CalculateRawRating().StarsRating); ActualWordsHandler.InstanceOpen.Container.Context.NGram = 3; review = ActualWordsHandler.InstanceOpen.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create(); Assert.AreEqual(5, review.CalculateRawRating().StarsRating); IRatingAdjustment adjustment = RatingAdjustment.Create(review, null); var resultDocument = new DocumentFromReviewFactory(ActualWordsHandler.InstanceOpen.Container.Resolve <INRCDictionary>()).ReparseDocument(adjustment); Assert.AreEqual(5, resultDocument.Stars); Assert.AreEqual("I Veto it really", resultDocument.Text); }
public async Task Adjusted() { ActualWordsHandler.InstanceOpen.Container.Context.DisableFeatureSentiment = true; var words = Path.Combine(TestContext.CurrentContext.TestDirectory, @"Adjustment/words.csv"); ISentimentDataHolder lexicon = SentimentDataHolder.Load(words); var text = "I Veto it"; var result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest(text)).ConfigureAwait(false); var document = result.Construct(ActualWordsHandler.InstanceOpen.WordFactory); ActualWordsHandler.InstanceOpen.Container.Context.Lexicon = lexicon; Text.Data.IParsedReview review = ActualWordsHandler.InstanceOpen.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create(); Assert.AreEqual(1, review.CalculateRawRating().StarsRating); }
public async Task AdjustSentiment(string word, int value, double?rating) { var request = await textSplitter.Process(new ParseRequest("Like or hate it")).ConfigureAwait(false); var document = request.Construct(ActualWordsHandler.InstanceSimple.WordFactory); var review = ActualWordsHandler.InstanceSimple.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create(); var adjustment = new LexiconRatingAdjustment( review, SentimentDataHolder.Load(new[] { new WordSentimentValueData(word, new SentimentValueData(value)), })); adjustment.CalculateRating(); Assert.AreEqual(rating, adjustment.Rating.RawRating); }
protected override Task Execute(CancellationToken token) { Logger.LogInformation("Initialize..."); container.Context.DisableFeatureSentiment = Config.InvertOff; Logger.LogInformation("Processing..."); ISentimentDataHolder sentimentAdjustment = default; if (!string.IsNullOrEmpty(Config.Weights)) { Logger.LogInformation("Adjusting Embeddings sentiments using [{0}] ...", Config.Weights); sentimentAdjustment = SentimentDataHolder.Load(Config.Weights); } IObservable <IParsedDocumentHolder> review = GetAllDocuments(); return(Process(review.Select(SynchronizedReviews).Merge(), container, sentimentAdjustment)); }
public async Task SentimentTests(SentimentTestData data) { log.LogInformation("SentimentTests: {0}", data); string file; switch (data.Category) { case ProductCategory.Electronics: file = "Electronics.csv"; break; case ProductCategory.Video: file = "video.csv"; break; case ProductCategory.Kitchen: file = "kitchen.csv"; break; case ProductCategory.Medic: case ProductCategory.Games: case ProductCategory.Toys: case ProductCategory.Book: throw new NotImplementedException(); default: throw new ArgumentOutOfRangeException(); } ISentimentDataHolder holder = SentimentDataHolder.Load(Path.Combine(TestContext.CurrentContext.TestDirectory, "Sentiments", file)); var runner = new TestRunner(TestHelper.Instance, data); Analysis.Processing.ITestingClient testing = runner.Active.GetTesting(); runner.Active.Context.Lexicon = holder; testing.DisableAspects = true; testing.DisableSvm = true; testing.TrackArff = true; testing.Init(); await testing.Process(runner.Load()).LastOrDefaultAsync(); testing.Save(Path.Combine(TestContext.CurrentContext.TestDirectory, "Word2Vec")); Assert.AreEqual(data.Performance, testing.GetPerformanceDescription()); Assert.AreEqual(data.Errors, testing.Errors); }
public async Task Process(IConnectionContext target, SentimentMessage message, CancellationToken token) { if (message == null) { throw new ArgumentNullException(nameof(message)); } var request = message.Request; if (request?.Documents == null) { throw new Exception("Nothing to process"); } if (request.Documents.Length > 500) { throw new Exception("Too many documents. Maximum is 500"); } var completed = new CompletedMessage(); try { var monitor = new PerformanceMonitor(request.Documents.Length); using (Observable.Interval(TimeSpan.FromSeconds(10)) .Subscribe(item => logger.LogInformation(monitor.ToString()))) { ISentimentDataHolder lexicon = default; if (request.Dictionary != null && request.Dictionary.Count > 0) { logger.LogInformation("Creating custom dictionary with {0} words", request.Dictionary.Count); lexicon = SentimentDataHolder.Load(request.Dictionary.Select(item => new WordSentimentValueData( item.Key, new SentimentValueData(item.Value)))); } if ((lexicon == null || request.AdjustDomain) && !string.IsNullOrEmpty(request.Domain)) { logger.LogInformation("Using Domain dictionary [{0}]", request.Domain); var previous = lexicon; lexicon = lexiconLoader.GetLexicon(request.Domain); if (previous != null) { lexicon.Merge(previous); } } string modelLocation = null; if (!string.IsNullOrEmpty(request.Model)) { logger.LogInformation("Using model path: {0}", request.Model); modelLocation = storage.GetLocation(target.Connection.User, request.Model, ServiceConstants.Model); if (!Directory.Exists(modelLocation)) { throw new ApplicationException($"Can't find model {request.Model}"); } } using (var scope = provider.CreateScope()) { var container = scope.ServiceProvider.GetService <ISessionContainer>(); container.Context.NGram = 3; container.Context.ExtractAttributes = request.Emotions; var client = container.GetTesting(modelLocation); var converter = scope.ServiceProvider.GetService <IDocumentConverter>(); client.Init(); client.Pipeline.ResetMonitor(); if (lexicon != null) { client.Lexicon = lexicon; } await client.Process(request.Documents.Select(item => converter.Convert(item, request.CleanText)) .ToObservable()) .Select(item => { monitor.Increment(); return(item); }) .Buffer(TimeSpan.FromSeconds(5), 10, scheduler) .Select(async item => { var result = new ResultMessage <Document> { Data = item.Select(x => x.Processed).ToArray() }; await target.Write(result, token).ConfigureAwait(false); return(Unit.Default); }) .Merge(); } logger.LogInformation("Completed with final performance: {0}", monitor); completed.Message = "Testing Completed"; await target.Write(completed, token).ConfigureAwait(false); } } catch (Exception e) { completed.Message = e.Message; await target.Write(completed, token).ConfigureAwait(false); completed.IsError = true; throw; } }