public async Task Process(IConnectionContext target, TrainMessage request, CancellationToken token)
{
if (target == null)
{
throw new ArgumentNullException(nameof(target));
}
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
ISentimentDataHolder loader = default;
var completed = new CompletedMessage();
try
{
if (!string.IsNullOrEmpty(request.Domain))
{
logger.LogInformation("Using Domain dictionary [{0}]", request.Domain);
loader = lexiconLoader.GetLexicon(request.Domain);
}
var modelLocation = storage.GetLocation(target.Connection.User, request.Name, ServiceConstants.Model);
using (var scope = provider.CreateScope())
{
var container = scope.ServiceProvider.GetService <ISessionContainer>();
container.Context.NGram = 3;
var client = container.GetTraining(modelLocation);
var converter = scope.ServiceProvider.GetService <IDocumentConverter>();
client.Pipeline.ResetMonitor();
if (loader != null)
{
client.Lexicon = loader;
}
var positive = storage.Load(target.Connection.User, request.Name, true)
.Take(2000);
var negative = storage.Load(target.Connection.User, request.Name, false)
.Take(2000);
var documents = positive.Concat(negative)
.Select(item => converter.Convert(item, request.CleanText));
await client.Train(documents).ConfigureAwait(false);
completed.Message = "Training Completed";
await target.Write(completed, token).ConfigureAwait(false);
}
}
catch (Exception e)
{
completed.Message = e.Message;
await target.Write(completed, token).ConfigureAwait(false);
completed.IsError = true;
throw;
}
}
public async Task Process(IConnectionContext target, SentimentMessage message, CancellationToken token)
{
if (message == null)
{
throw new ArgumentNullException(nameof(message));
}
var request = message.Request;
if (request?.Documents == null)
{
throw new Exception("Nothing to process");
}
if (request.Documents.Length > 500)
{
throw new Exception("Too many documents. Maximum is 500");
}
var completed = new CompletedMessage();
try
{
var monitor = new PerformanceMonitor(request.Documents.Length);
using (Observable.Interval(TimeSpan.FromSeconds(10))
.Subscribe(item => logger.LogInformation(monitor.ToString())))
{
ISentimentDataHolder lexicon = default;
if (request.Dictionary != null &&
request.Dictionary.Count > 0)
{
logger.LogInformation("Creating custom dictionary with {0} words", request.Dictionary.Count);
lexicon = SentimentDataHolder.Load(request.Dictionary.Select(item =>
new WordSentimentValueData(
item.Key,
new SentimentValueData(item.Value))));
}
if ((lexicon == null || request.AdjustDomain) &&
!string.IsNullOrEmpty(request.Domain))
{
logger.LogInformation("Using Domain dictionary [{0}]", request.Domain);
var previous = lexicon;
lexicon = lexiconLoader.GetLexicon(request.Domain);
if (previous != null)
{
lexicon.Merge(previous);
}
}
string modelLocation = null;
if (!string.IsNullOrEmpty(request.Model))
{
logger.LogInformation("Using model path: {0}", request.Model);
modelLocation = storage.GetLocation(target.Connection.User, request.Model, ServiceConstants.Model);
if (!Directory.Exists(modelLocation))
{
throw new ApplicationException($"Can't find model {request.Model}");
}
}
using (var scope = provider.CreateScope())
{
var container = scope.ServiceProvider.GetService <ISessionContainer>();
container.Context.NGram = 3;
container.Context.ExtractAttributes = request.Emotions;
var client = container.GetTesting(modelLocation);
var converter = scope.ServiceProvider.GetService <IDocumentConverter>();
client.Init();
client.Pipeline.ResetMonitor();
if (lexicon != null)
{
client.Lexicon = lexicon;
}
await client.Process(request.Documents.Select(item => converter.Convert(item, request.CleanText))
.ToObservable())
.Select(item =>
{
monitor.Increment();
return(item);
})
.Buffer(TimeSpan.FromSeconds(5), 10, scheduler)
.Select(async item =>
{
var result = new ResultMessage <Document> {
Data = item.Select(x => x.Processed).ToArray()
};
await target.Write(result, token).ConfigureAwait(false);
return(Unit.Default);
})
.Merge();
}
logger.LogInformation("Completed with final performance: {0}", monitor);
completed.Message = "Testing Completed";
await target.Write(completed, token).ConfigureAwait(false);
}
}
catch (Exception e)
{
completed.Message = e.Message;
await target.Write(completed, token).ConfigureAwait(false);
completed.IsError = true;
throw;
}
}
