public async Task Stores_In_Progress_Messages()
{
var generatedMessageA = new GeneratedMessage
{
StartTime = DateTimeOffset.UtcNow,
MessageId = Guid.NewGuid(),
MessageName = "MessageA",
};
var generatedMessageB = new GeneratedMessage
{
StartTime = DateTimeOffset.UtcNow,
MessageId = Guid.NewGuid(),
MessageName = "MessageB",
};
var jobStatusMessage = new RecordJobMessageProcessingStatus
{
Id = Guid.NewGuid(),
JobId = 1,
GeneratedMessages = new List <GeneratedMessage> {
generatedMessageA, generatedMessageB
},
EndTime = DateTime.UtcNow,
Succeeded = true
};
var service = mocker.Create <JobMessageService>();
await service.RecordCompletedJobMessageStatus(jobStatusMessage, CancellationToken.None);
mocker.Mock <IJobStorageService>()
.Verify(x => x.StoreInProgressMessages(It.Is <long>(jobId => jobId == jobStatusMessage.JobId), It.Is <List <InProgressMessage> >(identifiers =>
identifiers.Count == 2 &&
identifiers.Exists(inProgress => inProgress.MessageId == generatedMessageA.MessageId) &&
identifiers.Exists(inProgress => inProgress.MessageId == generatedMessageB.MessageId)), It.IsAny <CancellationToken>()), Times.Once);
}
public async Task ProcessedJobMessage(long jobId, Guid messageId, string messageName, List <GeneratedMessage> generatedMessages)
{
try
{
logger.LogVerbose($"Sending request to record successful processing of event. Job Id: {jobId}, Event: id: {messageId} ");
var itemProcessedEvent = new RecordJobMessageProcessingStatus
{
JobId = jobId,
Id = messageId,
MessageName = messageName,
EndTime = DateTimeOffset.UtcNow,
GeneratedMessages = generatedMessages ?? new List <GeneratedMessage>(),
Succeeded = true,
};
var partitionedEndpointName = config.GetMonitoringEndpointName(jobId);
await messageSession.Send(partitionedEndpointName, itemProcessedEvent).ConfigureAwait(false);
logger.LogDebug($"Sent request to record successful processing of event. Job Id: {jobId}, Event: id: {messageId} ");
}
catch (Exception ex)
{
logger.LogWarning($"Failed to send the job status message. Job: {jobId}, Message: {messageId}, {messageName}, Error: {ex.Message}, {ex}");
}
}
public async Task <JobStatus> RecordJobMessageProcessingStatus(RecordJobMessageProcessingStatus message, CancellationToken cancellationToken)
{
try
{
using (var operation = telemetry.StartOperation("JobsService.RecordJobMessageProcessingStatus", message.CommandId.ToString()))
{
var stopwatch = Stopwatch.StartNew();
//TODO: use statemanager tx and make sure IActorDataCache uses current tx too
var jobMessageService = lifetimeScope.Resolve <IJobMessageService>();
await jobMessageService.RecordCompletedJobMessageStatus(message, CancellationToken.None).ConfigureAwait(false);
var statusService = lifetimeScope.Resolve <IJobStatusService>();
var status = JobStatus.InProgress;
if (message.AllowJobCompletion)
{
status = await statusService.ManageStatus(CancellationToken.None);
if (status != JobStatus.InProgress)
{
await StateManager.ClearCacheAsync(CancellationToken.None);
}
}
telemetry.TrackDuration("JobsService.RecordJobMessageProcessingStatus", stopwatch.Elapsed);
telemetry.StopOperation(operation);
await SetLastMesasgeStatusTime().ConfigureAwait(false);
return(status);
}
}
catch (Exception e)
{
logger.LogWarning($"Error recording job message status. Job: {message.JobId}, message id: {message.Id}, name: {message.MessageName}. Error: {e.Message}. {e}");
throw;
}
}
public void SetUp()
{
jobMessageStatus = new RecordJobMessageProcessingStatus
{
JobId = 1,
MessageName = "Message1",
Id = Guid.NewGuid(),
Succeeded = true,
EndTime = DateTimeOffset.UtcNow,
GeneratedMessages = new List <GeneratedMessage>()
};
jobStep = new JobStepModel
{
JobId = 21,
StartTime = DateTimeOffset.UtcNow.AddSeconds(-10),
MessageName = "Message1",
MessageId = jobMessageStatus.Id,
Id = 2,
Status = JobStepStatus.Queued,
};
mocker = AutoMock.GetLoose();
var mockDataContext = mocker.Mock <IJobsDataContext>();
mockDataContext.Setup(x => x.GetJobIdFromDcJobId(It.IsAny <long>()))
.Returns(Task.FromResult <long>(99));
mockDataContext.Setup(x => x.GetJobByDcJobId(It.IsAny <long>()))
.Returns(Task.FromResult <JobModel>(new JobModel {
Id = jobStep.JobId, DcJobId = jobMessageStatus.JobId
}));
jobSteps = new List <JobStepModel> {
jobStep
};
mockDataContext
.Setup(dc => dc.GetJobSteps(It.IsAny <List <Guid> >()))
.Returns(Task.FromResult <List <JobStepModel> >(jobSteps));
mockDataContext.Setup(dc => dc.GetJobIdFromDcJobId(It.IsAny <long>()))
.Returns(Task.FromResult <long>(1));
object job = new JobModel {
Id = jobStep.JobId, DcJobId = jobMessageStatus.JobId
};
mocker.Mock <IMemoryCache>()
.Setup(cache => cache.TryGetValue(It.IsAny <string>(), out job))
.Returns(true);
}
public async Task Stores_Completed_Job_Message()
{
var jobStatusMessage = new RecordJobMessageProcessingStatus
{
Id = Guid.NewGuid(),
JobId = 1,
GeneratedMessages = new List <GeneratedMessage>(),
EndTime = DateTime.UtcNow,
Succeeded = true
};
var service = mocker.Create <JobMessageService>();
await service.RecordCompletedJobMessageStatus(jobStatusMessage, default(CancellationToken));
mocker.Mock <IJobStorageService>()
.Verify(
x => x.StoreCompletedMessage(It.Is <CompletedMessage>(msg =>
msg.MessageId == jobStatusMessage.Id && msg.CompletedTime == jobStatusMessage.EndTime), It.IsAny <CancellationToken>()),
Times.Once);
}
public async Task ProcessingFailedForJobMessage(byte[] failedMessageBody)
{
try
{
var messageJson = Encoding.UTF8.GetString(failedMessageBody).Trim(Encoding.UTF8.GetString(Encoding.UTF8.GetPreamble()).ToCharArray());
var message = JObject.Parse(messageJson);
var messageId = message.ContainsKey("EventId") ? (string)message["EventId"] : message.ContainsKey("CommandId") ? (string)message["CommandId"] : (string)null;
if (messageId == null)
{
logger.LogVerbose("No message id found on the message.");
return;
}
var job = message.ContainsKey("JobId") ? (string)message["JobId"] : (string)null;
if (job == null)
{
logger.LogVerbose("No job id found on the message.");
return;
}
if (!long.TryParse(job, out long jobId))
{
logger.LogVerbose($"No job id found on the message {messageId}.");
return;
}
var itemProcessedEvent = new RecordJobMessageProcessingStatus
{
JobId = jobId,
Id = Guid.Parse(messageId),
MessageName = string.Empty,
EndTime = DateTimeOffset.UtcNow,
GeneratedMessages = new List <GeneratedMessage>(),
Succeeded = false
};
var partitionedEndpointName = config.GetMonitoringEndpointName(jobId);
await messageSession.Send(partitionedEndpointName, itemProcessedEvent).ConfigureAwait(false);
}
catch (Exception ex)
{
logger.LogWarning($"Couldn't generate the job message failed message for monitoring. {ex.Message}");
}
}
public async Task RecordCompletedJobMessageStatus(RecordJobMessageProcessingStatus jobMessageStatus, CancellationToken cancellationToken)
{
var completedMessage = new CompletedMessage
{
MessageId = jobMessageStatus.Id, JobId = jobMessageStatus.JobId,
CompletedTime = jobMessageStatus.EndTime, Succeeded = jobMessageStatus.Succeeded
};
logger.LogVerbose($"Now storing the completed message. Message id: {completedMessage.MessageId}, Job: {completedMessage.JobId}, End time: {completedMessage.CompletedTime}, Succeeded: {completedMessage.Succeeded}");
await jobStorageService.StoreCompletedMessage(completedMessage, cancellationToken);
logger.LogVerbose($"Stored completed message. Now storing {jobMessageStatus.GeneratedMessages.Count} in progress messages generated while processing message: {completedMessage.MessageId} for job: {completedMessage.JobId}");
await jobStorageService.StoreInProgressMessages(jobMessageStatus.JobId,
jobMessageStatus.GeneratedMessages.Select(message => new InProgressMessage
{
MessageId = message.MessageId, JobId = jobMessageStatus.JobId, MessageName = message.MessageName
}).ToList(), cancellationToken);
logger.LogDebug($"Recorded completion of message processing. Job Id: {jobMessageStatus.JobId}, Message id: {jobMessageStatus.Id}.");
}
public async Task ProcessedJobMessage(long jobId, Guid messageId, string messageName, List <GeneratedMessage> generatedMessages)
{
try
{
logger.LogVerbose($"Sending request to record successful processing of event. Job Id: {jobId}, Event: id: {messageId} ");
var batchSize = 1000; //TODO: this should come from config
List <GeneratedMessage> batch;
var itemProcessedEvent = new RecordJobMessageProcessingStatus
{
JobId = jobId,
Id = messageId,
MessageName = messageName,
EndTime = DateTimeOffset.UtcNow,
GeneratedMessages = generatedMessages.Take(batchSize).ToList() ?? new List <GeneratedMessage>(),
Succeeded = true,
};
var partitionedEndpointName = config.GetMonitoringEndpointName(jobId);
await messageSession.Send(partitionedEndpointName, itemProcessedEvent).ConfigureAwait(false);
var skip = batchSize;
while ((batch = generatedMessages.Skip(skip).Take(batchSize).ToList()).Count > 0)
{
skip += batchSize;
var providerEarningsAdditionalMessages = new RecordJobAdditionalMessages
{
JobId = jobId,
GeneratedMessages = batch,
};
await messageSession.Send(partitionedEndpointName, providerEarningsAdditionalMessages).ConfigureAwait(false);
}
logger.LogDebug(
$"Sent request to record successful processing of event. Job Id: {jobId}, Event: id: {messageId} ");
}
catch (Exception ex)
{
logger.LogWarning($"Failed to send the job status message. Job: {jobId}, Message: {messageId}, {messageName}, Error: {ex.Message}, {ex}");
}
}
public void SetUp()
{
jobMessageStatus = new RecordJobMessageProcessingStatus
{
JobId = 1,
MessageName = "Message1",
Id = Guid.NewGuid(),
Succeeded = true,
EndTime = DateTimeOffset.UtcNow,
GeneratedMessages = new List <GeneratedMessage>()
};
jobStep = new JobStepModel
{
JobId = 21,
StartTime = DateTimeOffset.UtcNow.AddSeconds(-10),
MessageName = "Message1",
MessageId = jobMessageStatus.Id,
Id = 2,
Status = JobStepStatus.Queued,
};
jobSteps = new List <JobStepModel>();
mocker = AutoMock.GetLoose();
mocker.Mock <IJobStorageService>()
.Setup(x => x.StoreNewJob(It.IsAny <JobModel>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(true));
object job = new JobModel {
Id = jobStep.JobId, DcJobId = jobMessageStatus.JobId
};
mocker.Mock <IMemoryCache>()
.Setup(cache => cache.TryGetValue(It.IsAny <string>(), out job))
.Returns(true);
mocker.Mock <IJobStorageService>()
.Setup(x => x.GetInProgressMessages(It.IsAny <long>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(new List <InProgressMessage>());
}