public async Task Stores_Latest_Message_Completion_Time()
{
var jobId = 99;
var completedMessage = new CompletedMessage
{
MessageId = Guid.NewGuid(),
JobId = job.Id,
Succeeded = true,
CompletedTime = DateTimeOffset.UtcNow
};
var inProgressMessage = new InProgressMessage {
MessageId = completedMessage.MessageId
};
inProgressMessages.Add(inProgressMessage);
completedMessages.Add(completedMessage);
mocker.Mock <IJobStorageService>()
.Setup(x => x.GetJobStatus(It.IsAny <long>(), It.IsAny <CancellationToken>()))
.ReturnsAsync((hasFailedMessages: false, endTime: DateTimeOffset.UtcNow.AddSeconds(-10)));
var service = mocker.Create <EarningsJobStatusService>();
await service.ManageStatus(jobId, CancellationToken.None).ConfigureAwait(false);
mocker.Mock <IJobStorageService>()
.Verify(
x => x.StoreJobStatus(It.Is <long>(id => id == jobId), It.IsAny <bool>(),
It.Is <DateTimeOffset?>(endTime => endTime.Value == completedMessage.CompletedTime),
It.IsAny <CancellationToken>()), Times.Once);
}
public async Task Records_Earnings_Job_As_Completed_If_No_Learners()
{
var jobId = 99;
job.LearnerCount = 0;
var completedMessage = new CompletedMessage
{
MessageId = Guid.NewGuid(),
JobId = job.Id,
Succeeded = true,
CompletedTime = DateTimeOffset.UtcNow
};
var inProgressMessage = new InProgressMessage {
MessageId = completedMessage.MessageId, MessageName = "Message"
};
inProgressMessages.Add(inProgressMessage);
inProgressMessages.Add(new InProgressMessage {
MessageId = Guid.NewGuid(), MessageName = "Message"
});
completedMessages.Add(completedMessage);
var service = mocker.Create <JobStatusService>();
await service.ManageStatus(jobId, CancellationToken.None).ConfigureAwait(false);
mocker.Mock <IJobStorageService>()
.Verify(
x => x.SaveJobStatus(It.Is <long>(id => id == jobId),
It.Is <JobStatus>(status => status == JobStatus.Completed),
It.Is <DateTimeOffset>(endTime => endTime == job.StartTime),
It.IsAny <CancellationToken>()), Times.Once());
}
public async Task Does_Not_Return_Finished_If_All_Messages_Completed_But_DC_Not_Confirmed_Completed()
{
var jobId = 99;
job.DcJobEndTime = null;
job.DcJobSucceeded = null;
var completedMessage = new CompletedMessage
{
MessageId = Guid.NewGuid(),
JobId = job.Id,
Succeeded = true,
CompletedTime = DateTimeOffset.UtcNow
};
var inProgressMessage = new InProgressMessage
{
MessageId = completedMessage.MessageId, MessageName = "Message"
};
inProgressMessages.Add(inProgressMessage);
completedMessages.Add(completedMessage);
mocker.Mock <IJobStorageService>()
.Setup(x => x.GetJobStatus(It.IsAny <long>(), It.IsAny <CancellationToken>()))
.ReturnsAsync((hasFailedMessages: false, endTime: DateTimeOffset.UtcNow.AddSeconds(-10)));
var service = mocker.Create <EarningsJobStatusService>();
var finished = await service.ManageStatus(jobId, CancellationToken.None).ConfigureAwait(false);
finished.Should().BeFalse();
}
public async Task Publishes_PeriodEndJobFinished_with_SUCCESS_When_PeriodEndJob_Finishes(JobType jobType)
{
job.JobType = jobType;
job.DcJobSucceeded = null;
var completedMessage = new CompletedMessage
{
MessageId = Guid.NewGuid(),
JobId = job.Id,
Succeeded = true,
CompletedTime = DateTimeOffset.UtcNow
};
var inProgressMessage = new InProgressMessage {
MessageId = completedMessage.MessageId, MessageName = "Message"
};
inProgressMessages.Add(inProgressMessage);
completedMessages.Add(completedMessage);
mocker.Mock <IJobStorageService>()
.Setup(x => x.GetJobStatus(It.IsAny <long>(), It.IsAny <CancellationToken>()))
.ReturnsAsync((hasFailedMessages: false, endTime: DateTimeOffset.UtcNow.AddSeconds(-10)));
var service = mocker.Create <PeriodEndJobStatusService>();
await service.ManageStatus(job.Id, CancellationToken.None).ConfigureAwait(false);
mocker.Mock <IJobStatusEventPublisher>()
.Verify(publisher => publisher.PeriodEndJobFinished(It.Is <JobModel>(jobModel => jobModel == job)
, It.Is <bool>(b => b == true)), Times.Once);
mocker.Mock <IJobStatusEventPublisher>()
.Verify(publisher => publisher.PeriodEndJobFinished(It.Is <JobModel>(jobModel => jobModel == job)
, It.Is <bool>(b => b == false)), Times.Never);
}
public async Task Records_Job_Completed_With_Errors()
{
var jobId = 99;
var completedMessage = new CompletedMessage
{
MessageId = Guid.NewGuid(),
JobId = job.Id,
Succeeded = true,
CompletedTime = DateTimeOffset.UtcNow
};
var inProgressMessage = new InProgressMessage
{
MessageId = completedMessage.MessageId, MessageName = "Message"
};
inProgressMessages.Add(inProgressMessage);
completedMessages.Add(completedMessage);
mocker.Mock <IJobStorageService>()
.Setup(x => x.GetJobStatus(It.IsAny <long>(), It.IsAny <CancellationToken>()))
.ReturnsAsync((hasFailedMessages: true, endTime: DateTimeOffset.UtcNow.AddSeconds(-10)));
var service = mocker.Create <EarningsJobStatusService>();
await service.ManageStatus(jobId, CancellationToken.None).ConfigureAwait(false);
mocker.Mock <IJobStorageService>()
.Verify(
x => x.SaveJobStatus(It.Is <long>(id => id == jobId),
It.Is <JobStatus>(status => status == JobStatus.CompletedWithErrors),
It.Is <DateTimeOffset>(endTime => endTime == completedMessage.CompletedTime),
It.IsAny <CancellationToken>()), Times.Once());
}
public async Task Records_DataLocks_Completion_Time_After_Processing_DataLocks_Events(string messageName)
{
var jobId = 99;
var completedMessage = new CompletedMessage
{
MessageId = Guid.NewGuid(),
JobId = job.Id,
Succeeded = true,
CompletedTime = DateTimeOffset.UtcNow,
};
var inProgressMessage = new InProgressMessage
{
MessageName = messageName, MessageId = completedMessage.MessageId
};
inProgressMessages.Add(inProgressMessage);
completedMessages.Add(completedMessage);
mocker.Mock <IJobStorageService>()
.Setup(x => x.GetJobStatus(It.IsAny <long>(), It.IsAny <CancellationToken>()))
.ReturnsAsync((hasFailedMessages: false, endTime: DateTimeOffset.UtcNow.AddSeconds(-10)));
var service = mocker.Create <EarningsJobStatusService>();
await service.ManageStatus(jobId, CancellationToken.None).ConfigureAwait(false);
mocker.Mock <IJobStorageService>()
.Verify(
x => x.SaveDataLocksCompletionTime(It.Is <long>(id => id == jobId),
It.Is <DateTimeOffset>(endTime => endTime == completedMessage.CompletedTime),
It.IsAny <CancellationToken>()), Times.Once());
}
public async Task Does_Not_record_Job_As_Completed_If_No_Learners_And_Not_Earnings_Job()
{
var jobId = 99;
job.LearnerCount = 0;
job.JobType = JobType.PeriodEndRunJob;
var completedMessage = new CompletedMessage
{
MessageId = Guid.NewGuid(),
JobId = job.Id,
Succeeded = true,
CompletedTime = DateTimeOffset.UtcNow
};
var inProgressMessage = new InProgressMessage {
MessageId = completedMessage.MessageId, MessageName = "Message"
};
inProgressMessages.Add(inProgressMessage);
inProgressMessages.Add(new InProgressMessage {
MessageId = Guid.NewGuid(), MessageName = "Message"
});
completedMessages.Add(completedMessage);
var service = mocker.Create <JobStatusService>();
await service.ManageStatus(jobId, CancellationToken.None).ConfigureAwait(false);
mocker.Mock <IJobStorageService>()
.Verify(
x => x.SaveJobStatus(It.IsAny <long>(),
It.IsAny <JobStatus>(),
It.IsAny <DateTimeOffset>(),
It.IsAny <CancellationToken>()), Times.Never);
}
// TODO, do some pooling here, for performance
public void AddData(byte[] buffer, object tag)
{
if (PartialMessage == null)
{
PartialMessage = buffer;
}
else if (!OneMessagePerPacket) // can't add packets together
{
int copyStart = PartialMessage.Length;
Array.Resize(ref PartialMessage, copyStart + buffer.Length);
Array.Copy(buffer, 0, PartialMessage, copyStart, buffer.Length);
}
if (PartialMessage.Length >= 4)
{
int len = BufferUtils.ReadUInt16(PartialMessage, 0);
int code = BufferUtils.ReadUInt16(PartialMessage, 2);
if (len > MaxSanityBuffer)
{
// odd shit, just assume the new packet is a buffer
PartialMessage = buffer;
len = BufferUtils.ReadUInt16(PartialMessage, 0);
code = BufferUtils.ReadUInt16(PartialMessage, 2);
if (PartialMessage.Length < 4)
{
return;
}
}
if (PartialMessage.Length >= (len + 4))
{
// message is long enough, parse it
CompletedMessage msg = new CompletedMessage();
msg.ID = code;
msg.Size = len;
msg.Tag = tag;
msg.Data = new byte[len];
Array.Copy(PartialMessage, 4, msg.Data, 0, len);
string msgCode = Encoding.ASCII.GetString(PartialMessage, 2, 2);
PushMessage(msg);
if (OneMessagePerPacket || PartialMessage.Length == len + 4) // one message per packet
{
PartialMessage = null;
}
else
{
byte[] remanats = new byte[PartialMessage.Length - (len + 4)];
Array.Copy(PartialMessage, (len + 4), remanats, 0, remanats.Length);
PartialMessage = remanats;
}
}
}
else if (OneMessagePerPacket) // one message per packet
{
PartialMessage = null;
}
}
public async Task Publishes_JobFinished_When_Job_Finished_And_Recorded_DC_Failure()
{
job.DcJobSucceeded = false;
var completedMessage = new CompletedMessage
{
MessageId = Guid.NewGuid(),
JobId = job.Id,
Succeeded = true,
CompletedTime = DateTimeOffset.UtcNow
};
var inProgressMessage = new InProgressMessage
{
MessageId = completedMessage.MessageId, MessageName = "Message"
};
inProgressMessages.Add(inProgressMessage);
completedMessages.Add(completedMessage);
mocker.Mock <IJobStorageService>()
.Setup(x => x.GetJobStatus(It.IsAny <long>(), It.IsAny <CancellationToken>()))
.ReturnsAsync((hasFailedMessages: false, endTime: DateTimeOffset.UtcNow.AddSeconds(-10)));
var service = mocker.Create <EarningsJobStatusService>();
await service.ManageStatus(job.Id, CancellationToken.None).ConfigureAwait(false);
mocker.Mock <IJobStatusEventPublisher>()
.Verify(publisher => publisher.SubmissionFinished(It.Is <bool>(succeeded => !succeeded),
It.Is <long>(id => id == job.DcJobId),
It.Is <long>(ukprn => ukprn == job.Ukprn),
It.Is <short>(academicYEar => academicYEar == job.AcademicYear),
It.Is <byte>(collectionPeriod => collectionPeriod == job.CollectionPeriod),
It.Is <DateTime>(ilrSubmissionTIme => ilrSubmissionTIme == job.IlrSubmissionTime)), Times.Once);
}
public async Task Process(IConnectionContext target, TrainMessage request, CancellationToken token)
{
if (target == null)
{
throw new ArgumentNullException(nameof(target));
}
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
ISentimentDataHolder loader = default;
var completed = new CompletedMessage();
try
{
if (!string.IsNullOrEmpty(request.Domain))
{
logger.LogInformation("Using Domain dictionary [{0}]", request.Domain);
loader = lexiconLoader.GetLexicon(request.Domain);
}
var modelLocation = storage.GetLocation(target.Connection.User, request.Name, ServiceConstants.Model);
using (var scope = provider.CreateScope())
{
var container = scope.ServiceProvider.GetService <ISessionContainer>();
container.Context.NGram = 3;
var client = container.GetTraining(modelLocation);
var converter = scope.ServiceProvider.GetService <IDocumentConverter>();
client.Pipeline.ResetMonitor();
if (loader != null)
{
client.Lexicon = loader;
}
var positive = storage.Load(target.Connection.User, request.Name, true)
.Take(2000);
var negative = storage.Load(target.Connection.User, request.Name, false)
.Take(2000);
var documents = positive.Concat(negative)
.Select(item => converter.Convert(item, request.CleanText));
await client.Train(documents).ConfigureAwait(false);
completed.Message = "Training Completed";
await target.Write(completed, token).ConfigureAwait(false);
}
}
catch (Exception e)
{
completed.Message = e.Message;
await target.Write(completed, token).ConfigureAwait(false);
completed.IsError = true;
throw;
}
}
public void AddCompletedCommandToBatch(Guid batchId, Guid messageId)
{
using (var client = GetClient())
{
var completedMessage = new CompletedMessage {
BatchId = batchId, MessageId = messageId
};
client.PostAsJsonAsync("CompleteMessage", completedMessage).RunSynchronously();
}
}
public void CompleteMessage(CompletedMessage message)
{
if (!_completedItems.ContainsKey(message.BatchId))
{
_completedItems.Add(message.BatchId, new List <Guid>());
}
_completedItems[message.BatchId].Add(message.MessageId);
Console.WriteLine($"Message {message.MessageId} marked as completed.");
}
public async Task StoreCompletedMessage(CompletedMessage completedMessage, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var completedMessagesCollection =
await GetCompletedMessagesCollection(completedMessage.JobId).ConfigureAwait(false);
await completedMessagesCollection.AddOrUpdateAsync(reliableTransactionProvider.Current,
completedMessage.MessageId,
completedMessage, (key, value) => completedMessage, TransactionTimeout, cancellationToken)
.ConfigureAwait(false);
}
protected void PushMessage(CompletedMessage msg)
{
lock (CompletedMessages)
{
CompletedMessages.Add(msg);
}
if (CompleteMessageRecived != null)
{
CompleteMessageRecived.Invoke(this, EventArgs.Empty);
}
}
public CompletedMessage GetMessage()
{
CompletedMessage msg = null;
lock (CompletedMessages)
{
if (CompletedMessages.Count != 0)
{
msg = CompletedMessages[0];
CompletedMessages.RemoveAt(0);
}
}
return(msg);
}
public async Task Publishes_PeriodEndJobFinished_with_SUCCESS_When_All_OutstandingJobs_are_Completed()
{
job.JobType = JobType.PeriodEndStartJob;
job.DcJobSucceeded = null;
var completedMessage = new CompletedMessage
{
MessageId = Guid.NewGuid(),
JobId = job.Id,
Succeeded = true,
CompletedTime = DateTimeOffset.UtcNow
};
var inProgressMessage = new InProgressMessage {
MessageId = completedMessage.MessageId, MessageName = "Message"
};
inProgressMessages.Add(inProgressMessage);
completedMessages.Add(completedMessage);
mocker.Mock <IJobStorageService>()
.Setup(x => x.GetJobStatus(It.IsAny <long>(), It.IsAny <CancellationToken>()))
.ReturnsAsync((hasFailedMessages: false, endTime: DateTimeOffset.UtcNow.AddSeconds(-10)));
mocker.Mock <IJobsDataContext>()
.Setup(x => x.GetOutstandingOrTimedOutJobs(It.IsAny <JobModel>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(new List <OutstandingJobResult>
{
new OutstandingJobResult {
DcJobId = 1, DcJobSucceeded = true, JobStatus = JobStatus.Completed
}
});
mocker.Mock <IJobsDataContext>()
.Setup(x => x.DoSubmissionSummariesExistForJobs(It.IsAny <List <OutstandingJobResult> >()))
.Returns(new List <long?>());
var service = mocker.Create <PeriodEndStartJobStatusService>();
await service.ManageStatus(job.Id, CancellationToken.None).ConfigureAwait(false);
mocker.Mock <IJobStatusEventPublisher>()
.Verify(publisher => publisher.PeriodEndJobFinished(It.Is <JobModel>(jobModel => jobModel == job)
, It.Is <bool>(b => b == true)), Times.Once);
mocker.Mock <IJobStatusEventPublisher>()
.Verify(publisher => publisher.PeriodEndJobFinished(It.Is <JobModel>(jobModel => jobModel == job)
, It.Is <bool>(b => b == false)), Times.Never);
}
public async Task Returns_False_And_Does_Not_Publishes_PeriodEndJobFinished_When_DAS_InProgress_DCStatus_Null_OutstandingJobs_found()
{
job.JobType = JobType.PeriodEndStartJob;
job.DcJobSucceeded = null;
var completedMessage = new CompletedMessage
{
MessageId = Guid.NewGuid(),
JobId = job.Id,
Succeeded = true,
CompletedTime = DateTimeOffset.UtcNow
};
var inProgressMessage = new InProgressMessage {
MessageId = completedMessage.MessageId, MessageName = "Message"
};
inProgressMessages.Add(inProgressMessage);
completedMessages.Add(completedMessage);
mocker.Mock <IJobStorageService>()
.Setup(x => x.GetJobStatus(It.IsAny <long>(), It.IsAny <CancellationToken>()))
.ReturnsAsync((hasFailedMessages: false, endTime: DateTimeOffset.UtcNow.AddSeconds(-10)));
mocker.Mock <IJobsDataContext>()
.Setup(x => x.GetOutstandingOrTimedOutJobs(It.IsAny <JobModel>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(new List <OutstandingJobResult>
{
new OutstandingJobResult {
DcJobId = 1, DcJobSucceeded = null, JobStatus = JobStatus.InProgress
}
});
var service = mocker.Create <PeriodEndStartJobStatusService>();
var result = await service.ManageStatus(job.Id, CancellationToken.None).ConfigureAwait(false);
mocker.Mock <IJobStatusEventPublisher>()
.Verify(publisher => publisher.PeriodEndJobFinished(It.Is <JobModel>(jobModel => jobModel == job),
It.IsAny <bool>()), Times.Never);
result.Should().BeFalse();
mocker.Mock <ITelemetry>().Verify(t => t.TrackEvent(
"PeriodEndStart Job Status Update",
It.Is <Dictionary <string, string> >(d =>
d.Contains(new KeyValuePair <string, string>("InProgressJobsCount", "1"))),
It.IsAny <Dictionary <string, double> >()));
}
public async Task RecordCompletedJobMessageStatus(RecordJobMessageProcessingStatus jobMessageStatus, CancellationToken cancellationToken)
{
var completedMessage = new CompletedMessage
{
MessageId = jobMessageStatus.Id, JobId = jobMessageStatus.JobId,
CompletedTime = jobMessageStatus.EndTime, Succeeded = jobMessageStatus.Succeeded
};
logger.LogVerbose($"Now storing the completed message. Message id: {completedMessage.MessageId}, Job: {completedMessage.JobId}, End time: {completedMessage.CompletedTime}, Succeeded: {completedMessage.Succeeded}");
await jobStorageService.StoreCompletedMessage(completedMessage, cancellationToken);
logger.LogVerbose($"Stored completed message. Now storing {jobMessageStatus.GeneratedMessages.Count} in progress messages generated while processing message: {completedMessage.MessageId} for job: {completedMessage.JobId}");
await jobStorageService.StoreInProgressMessages(jobMessageStatus.JobId,
jobMessageStatus.GeneratedMessages.Select(message => new InProgressMessage
{
MessageId = message.MessageId, JobId = jobMessageStatus.JobId, MessageName = message.MessageName
}).ToList(), cancellationToken);
logger.LogDebug($"Recorded completion of message processing. Job Id: {jobMessageStatus.JobId}, Message id: {jobMessageStatus.Id}.");
}
protected override void CompleteDialogExtention()
{
ObjectToEnter.HideActive = false;
//uh huh - okay now
ObjectToEnter.Active = true;
//lets set the connection in the service our parent dialog is using
if (XrmRecordService != null)
{
XrmRecordService.XrmRecordConfiguration = ObjectToEnter;
}
//lets also refresh it in the applications containers
SavedXrmConnectionsModule.RefreshXrmServices(ObjectToEnter, ApplicationController, xrmRecordService: XrmRecordService);
//lets also refresh it in the saved settings
var appSettingsManager = ApplicationController.ResolveType(typeof(ISettingsManager)) as ISettingsManager;
var savedConnectionsObject = ApplicationController.ResolveType <ISavedXrmConnections>();
if (savedConnectionsObject.Connections != null)
{
foreach (var item in savedConnectionsObject.Connections)
{
item.Active = false;
}
}
savedConnectionsObject.Connections
= savedConnectionsObject.Connections == null
? new [] { ObjectToEnter }
: savedConnectionsObject.Connections.Union(new [] { ObjectToEnter }).ToArray();
appSettingsManager.SaveSettingsObject(savedConnectionsObject);
var recordconfig =
new ObjectMapping.ClassMapperFor <SavedXrmRecordConfiguration, XrmRecordConfiguration>().Map(ObjectToEnter);
appSettingsManager.SaveSettingsObject(recordconfig);
if (!HasParentDialog)
{
CompletionItem = new CompletedMessage();
}
}
/// <summary>
/// Sends a message to a recipient via Facebook Messenger.
/// </summary>
/// <param name="recipient">The message recipient. Must be a valid <see cref="Recipient"/> subclass.</param>
/// <param name="message">The completed message.</param>
/// <param name="messageType">The message type. Defaults to <see cref="MessageType.Standard"/>.</param>
/// <param name="messagingType">The messaging type. Defaults to <see cref="MessagingType.Response"/>.</param>
/// <param name="notificationType">The notification type. Defaults to <see cref="NotificationType.Regular"/>.</param>
/// <param name="tag">An optional message reference tag.</param>
/// <returns>The id of the sent message.</returns>
public async Task <string> SendMessage(
Recipient recipient,
CompletedMessage message,
MessageType messageType = MessageType.Standard,
MessagingType messagingType = MessagingType.Response,
NotificationType notificationType = NotificationType.Regular,
string tag = default
)
{
//todo: split messages every 2000 characters
string messagingTypeString;
switch (messagingType)
{
case MessagingType.Response:
messagingTypeString = "RESPONSE";
break;
case MessagingType.MessageTag:
messagingTypeString = "MESSAGE_TAG";
break;
case MessagingType.Update:
messagingTypeString = "UPDATE";
break;
default:
throw new MessagingTypeNotSupportedException(messagingType);
}
string notificationTypeString;
switch (notificationType)
{
case NotificationType.Regular:
notificationTypeString = "REGULAR";
break;
case NotificationType.SilentPush:
notificationTypeString = "SILENT_PUSH";
break;
case NotificationType.NoPush:
notificationTypeString = "NO_PUSH";
break;
default:
throw new MessagingTypeNotSupportedException(messagingType);
}
var requestContainer = new SendRequestContainerEntity
{
MessagingType = messagingTypeString,
Recipient = recipient.ToEntity(),
NotificationType = notificationTypeString,
Message = message.GetMessageEntity()
};
if (tag != default)
{
requestContainer.Tag = tag;
}
if (messageType == MessageType.Subscription)
{
requestContainer.Tag = "NON_PROMOTIONAL_SUBSCRIPTION";
}
var result = await _client.Post <SendMessageResponse, MessengerErrorWrapperEntity>(ApiUri.ToString(), requestContainer);
if (result.Error != null)
{
throw new MessengerRequestFailedException("text", ApiUri.GetLeftPart(UriPartial.Path), result.HttpCode, result.Error.MessengerError.Code, result.Error.MessengerError.Message, result.Error.MessengerError.TraceId, result.Error.MessengerError.SubCode);
}
return(result.Success.MessageId);
}
private static InProgressMessage CreateInProgressMessage(CompletedMessage completedMessage)
{
return(new InProgressMessage {
MessageId = completedMessage.MessageId, MessageName = "Message"
});
}
public async Task Process(IConnectionContext target, SentimentMessage message, CancellationToken token)
{
if (message == null)
{
throw new ArgumentNullException(nameof(message));
}
var request = message.Request;
if (request?.Documents == null)
{
throw new Exception("Nothing to process");
}
if (request.Documents.Length > 500)
{
throw new Exception("Too many documents. Maximum is 500");
}
var completed = new CompletedMessage();
try
{
var monitor = new PerformanceMonitor(request.Documents.Length);
using (Observable.Interval(TimeSpan.FromSeconds(10))
.Subscribe(item => logger.LogInformation(monitor.ToString())))
{
ISentimentDataHolder lexicon = default;
if (request.Dictionary != null &&
request.Dictionary.Count > 0)
{
logger.LogInformation("Creating custom dictionary with {0} words", request.Dictionary.Count);
lexicon = SentimentDataHolder.Load(request.Dictionary.Select(item =>
new WordSentimentValueData(
item.Key,
new SentimentValueData(item.Value))));
}
if ((lexicon == null || request.AdjustDomain) &&
!string.IsNullOrEmpty(request.Domain))
{
logger.LogInformation("Using Domain dictionary [{0}]", request.Domain);
var previous = lexicon;
lexicon = lexiconLoader.GetLexicon(request.Domain);
if (previous != null)
{
lexicon.Merge(previous);
}
}
string modelLocation = null;
if (!string.IsNullOrEmpty(request.Model))
{
logger.LogInformation("Using model path: {0}", request.Model);
modelLocation = storage.GetLocation(target.Connection.User, request.Model, ServiceConstants.Model);
if (!Directory.Exists(modelLocation))
{
throw new ApplicationException($"Can't find model {request.Model}");
}
}
using (var scope = provider.CreateScope())
{
var container = scope.ServiceProvider.GetService <ISessionContainer>();
container.Context.NGram = 3;
container.Context.ExtractAttributes = request.Emotions;
var client = container.GetTesting(modelLocation);
var converter = scope.ServiceProvider.GetService <IDocumentConverter>();
client.Init();
client.Pipeline.ResetMonitor();
if (lexicon != null)
{
client.Lexicon = lexicon;
}
await client.Process(request.Documents.Select(item => converter.Convert(item, request.CleanText))
.ToObservable())
.Select(item =>
{
monitor.Increment();
return(item);
})
.Buffer(TimeSpan.FromSeconds(5), 10, scheduler)
.Select(async item =>
{
var result = new ResultMessage <Document> {
Data = item.Select(x => x.Processed).ToArray()
};
await target.Write(result, token).ConfigureAwait(false);
return(Unit.Default);
})
.Merge();
}
logger.LogInformation("Completed with final performance: {0}", monitor);
completed.Message = "Testing Completed";
await target.Write(completed, token).ConfigureAwait(false);
}
}
catch (Exception e)
{
completed.Message = e.Message;
await target.Write(completed, token).ConfigureAwait(false);
completed.IsError = true;
throw;
}
}
