public async Task NgramSentiment()
{
ActualWordsHandler.InstanceOpen.Container.Context.DisableFeatureSentiment = true;
var words = Path.Combine(TestContext.CurrentContext.TestDirectory, @"Adjustment/words.csv");
ISentimentDataHolder lexicon = SentimentDataHolder.Load(words);
var loader = SentimentDataHolder.Load(new[] { "veto it really" }.Select(item =>
new WordSentimentValueData(
item,
new SentimentValueData(2))));
lexicon.Merge(loader);
var text = "I Veto it really";
var result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest(text)).ConfigureAwait(false);
var document = result.Construct(ActualWordsHandler.InstanceOpen.WordFactory);
ActualWordsHandler.InstanceOpen.Container.Context.Lexicon = lexicon;
Text.Data.IParsedReview review = ActualWordsHandler.InstanceOpen.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create();
Assert.AreEqual(1, review.CalculateRawRating().StarsRating);
ActualWordsHandler.InstanceOpen.Container.Context.NGram = 3;
review = ActualWordsHandler.InstanceOpen.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create();
Assert.AreEqual(5, review.CalculateRawRating().StarsRating);
IRatingAdjustment adjustment = RatingAdjustment.Create(review, null);
var resultDocument = new DocumentFromReviewFactory(ActualWordsHandler.InstanceOpen.Container.Resolve <INRCDictionary>()).ReparseDocument(adjustment);
Assert.AreEqual(5, resultDocument.Stars);
Assert.AreEqual("I Veto it really", resultDocument.Text);
}
public async Task Adjusted()
{
ActualWordsHandler.InstanceOpen.Container.Context.DisableFeatureSentiment = true;
var words = Path.Combine(TestContext.CurrentContext.TestDirectory, @"Adjustment/words.csv");
ISentimentDataHolder lexicon = SentimentDataHolder.Load(words);
var text = "I Veto it";
var result = await ActualWordsHandler.InstanceOpen.TextSplitter.Process(new ParseRequest(text)).ConfigureAwait(false);
var document = result.Construct(ActualWordsHandler.InstanceOpen.WordFactory);
ActualWordsHandler.InstanceOpen.Container.Context.Lexicon = lexicon;
Text.Data.IParsedReview review = ActualWordsHandler.InstanceOpen.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create();
Assert.AreEqual(1, review.CalculateRawRating().StarsRating);
}
public async Task AdjustSentiment(string word, int value, double?rating)
{
var request = await textSplitter.Process(new ParseRequest("Like or hate it")).ConfigureAwait(false);
var document = request.Construct(ActualWordsHandler.InstanceSimple.WordFactory);
var review = ActualWordsHandler.InstanceSimple.Container.Resolve <Func <Document, IParsedReviewManager> >()(document).Create();
var adjustment = new LexiconRatingAdjustment(
review,
SentimentDataHolder.Load(new[]
{
new WordSentimentValueData(word, new SentimentValueData(value)),
}));
adjustment.CalculateRating();
Assert.AreEqual(rating, adjustment.Rating.RawRating);
}
protected override Task Execute(CancellationToken token)
{
Logger.LogInformation("Initialize...");
container.Context.DisableFeatureSentiment = Config.InvertOff;
Logger.LogInformation("Processing...");
ISentimentDataHolder sentimentAdjustment = default;
if (!string.IsNullOrEmpty(Config.Weights))
{
Logger.LogInformation("Adjusting Embeddings sentiments using [{0}] ...", Config.Weights);
sentimentAdjustment = SentimentDataHolder.Load(Config.Weights);
}
IObservable <IParsedDocumentHolder> review = GetAllDocuments();
return(Process(review.Select(SynchronizedReviews).Merge(), container, sentimentAdjustment));
}
public async Task SentimentTests(SentimentTestData data)
{
log.LogInformation("SentimentTests: {0}", data);
string file;
switch (data.Category)
{
case ProductCategory.Electronics:
file = "Electronics.csv";
break;
case ProductCategory.Video:
file = "video.csv";
break;
case ProductCategory.Kitchen:
file = "kitchen.csv";
break;
case ProductCategory.Medic:
case ProductCategory.Games:
case ProductCategory.Toys:
case ProductCategory.Book:
throw new NotImplementedException();
default:
throw new ArgumentOutOfRangeException();
}
ISentimentDataHolder holder = SentimentDataHolder.Load(Path.Combine(TestContext.CurrentContext.TestDirectory, "Sentiments", file));
var runner = new TestRunner(TestHelper.Instance, data);
Analysis.Processing.ITestingClient testing = runner.Active.GetTesting();
runner.Active.Context.Lexicon = holder;
testing.DisableAspects = true;
testing.DisableSvm = true;
testing.TrackArff = true;
testing.Init();
await testing.Process(runner.Load()).LastOrDefaultAsync();
testing.Save(Path.Combine(TestContext.CurrentContext.TestDirectory, "Word2Vec"));
Assert.AreEqual(data.Performance, testing.GetPerformanceDescription());
Assert.AreEqual(data.Errors, testing.Errors);
}
public async Task Process(IConnectionContext target, SentimentMessage message, CancellationToken token)
{
if (message == null)
{
throw new ArgumentNullException(nameof(message));
}
var request = message.Request;
if (request?.Documents == null)
{
throw new Exception("Nothing to process");
}
if (request.Documents.Length > 500)
{
throw new Exception("Too many documents. Maximum is 500");
}
var completed = new CompletedMessage();
try
{
var monitor = new PerformanceMonitor(request.Documents.Length);
using (Observable.Interval(TimeSpan.FromSeconds(10))
.Subscribe(item => logger.LogInformation(monitor.ToString())))
{
ISentimentDataHolder lexicon = default;
if (request.Dictionary != null &&
request.Dictionary.Count > 0)
{
logger.LogInformation("Creating custom dictionary with {0} words", request.Dictionary.Count);
lexicon = SentimentDataHolder.Load(request.Dictionary.Select(item =>
new WordSentimentValueData(
item.Key,
new SentimentValueData(item.Value))));
}
if ((lexicon == null || request.AdjustDomain) &&
!string.IsNullOrEmpty(request.Domain))
{
logger.LogInformation("Using Domain dictionary [{0}]", request.Domain);
var previous = lexicon;
lexicon = lexiconLoader.GetLexicon(request.Domain);
if (previous != null)
{
lexicon.Merge(previous);
}
}
string modelLocation = null;
if (!string.IsNullOrEmpty(request.Model))
{
logger.LogInformation("Using model path: {0}", request.Model);
modelLocation = storage.GetLocation(target.Connection.User, request.Model, ServiceConstants.Model);
if (!Directory.Exists(modelLocation))
{
throw new ApplicationException($"Can't find model {request.Model}");
}
}
using (var scope = provider.CreateScope())
{
var container = scope.ServiceProvider.GetService <ISessionContainer>();
container.Context.NGram = 3;
container.Context.ExtractAttributes = request.Emotions;
var client = container.GetTesting(modelLocation);
var converter = scope.ServiceProvider.GetService <IDocumentConverter>();
client.Init();
client.Pipeline.ResetMonitor();
if (lexicon != null)
{
client.Lexicon = lexicon;
}
await client.Process(request.Documents.Select(item => converter.Convert(item, request.CleanText))
.ToObservable())
.Select(item =>
{
monitor.Increment();
return(item);
})
.Buffer(TimeSpan.FromSeconds(5), 10, scheduler)
.Select(async item =>
{
var result = new ResultMessage <Document> {
Data = item.Select(x => x.Processed).ToArray()
};
await target.Write(result, token).ConfigureAwait(false);
return(Unit.Default);
})
.Merge();
}
logger.LogInformation("Completed with final performance: {0}", monitor);
completed.Message = "Testing Completed";
await target.Write(completed, token).ConfigureAwait(false);
}
}
catch (Exception e)
{
completed.Message = e.Message;
await target.Write(completed, token).ConfigureAwait(false);
completed.IsError = true;
throw;
}
}
