/// <inheritdoc/>
public async Task <HeartbeatResultModel> SendHeartbeatAsync(HeartbeatModel heartbeat,
JobDiagnosticInfoModel info, CancellationToken ct)
{
if (heartbeat == null)
{
throw new ArgumentNullException(nameof(heartbeat));
}
while (true)
{
var uri = _config?.Config?.JobOrchestratorUrl?.TrimEnd('/');
if (uri == null)
{
throw new InvalidConfigurationException("Job orchestrator not configured");
}
var request = _httpClient.NewRequest($"{uri}/v2/heartbeat");
request.Headers.Authorization = new AuthenticationHeaderValue("Basic",
_tokenProvider.IdentityToken.ToAuthorizationValue());
_serializer.SerializeToRequest(request, heartbeat.ToApiModel());
var response = await _httpClient.PostAsync(request, ct)
.ConfigureAwait(false);
try {
response.Validate();
var result = _serializer.DeserializeResponse <HeartbeatResponseApiModel>(
response);
return(result.ToServiceModel());
}
catch (UnauthorizedAccessException) {
await _tokenProvider.ForceUpdate();
}
}
}
/// <inheritdoc/>
public JobDiagnosticInfoModel GetDiagnosticInfo()
{
var totalSeconds = (DateTime.UtcNow - _diagnosticStart).TotalSeconds;
double totalDuration = _diagnosticStart != DateTime.MinValue ? totalSeconds : 0;
double chunkSizeAverage = _messageEncoder.AvgMessageSize / (4 * 1024);
double sentMessagesPerSec = totalDuration > 0 ? _messageSink.SentMessagesCount / totalDuration : 0;
double estimatedMsgChunksPerDay = Math.Ceiling(chunkSizeAverage) * sentMessagesPerSec * 60 * 60 * 24;
var diagnosticInfo = new JobDiagnosticInfoModel();
var endpointDiagnosticInfo = new EndpointDiagnosticModel();
endpointDiagnosticInfo.EndpointUrl = _messageTrigger.EndpointUrl;
endpointDiagnosticInfo.DataSetWriterGroup = _messageTrigger.DataSetWriterGroup;
endpointDiagnosticInfo.UseSecurity = _messageTrigger.UseSecurity;
endpointDiagnosticInfo.OpcAuthenticationMode = (AuthMode)_messageTrigger.AuthenticationMode;
endpointDiagnosticInfo.OpcAuthenticationUsername = _messageTrigger.AuthenticationUsername;
diagnosticInfo.Endpoint = endpointDiagnosticInfo;
diagnosticInfo.Id = Name;
diagnosticInfo.SentMessagesPerSec = sentMessagesPerSec;
diagnosticInfo.IngestionDuration = TimeSpan.FromSeconds(totalDuration);
diagnosticInfo.IngressDataChanges = _messageTrigger.DataChangesCount;
diagnosticInfo.IngressValueChanges = _messageTrigger.ValueChangesCount;
diagnosticInfo.IngressBatchBlockBufferSize = _batchDataSetMessageBlock.OutputCount;
diagnosticInfo.EncodingBlockInputSize = _encodingBlock.InputCount;
diagnosticInfo.EncodingBlockOutputSize = _encodingBlock.OutputCount;
diagnosticInfo.EncoderNotificationsProcessed = _messageEncoder.NotificationsProcessedCount;
diagnosticInfo.EncoderNotificationsDropped = _messageEncoder.NotificationsDroppedCount;
diagnosticInfo.EncoderIoTMessagesProcessed = _messageEncoder.MessagesProcessedCount;
diagnosticInfo.EncoderAvgNotificationsMessage = _messageEncoder.AvgNotificationsPerMessage;
diagnosticInfo.EncoderAvgIoTMessageBodySize = _messageEncoder.AvgMessageSize;
diagnosticInfo.EncoderAvgIoTChunkUsage = chunkSizeAverage;
diagnosticInfo.EstimatedIoTChunksPerDay = estimatedMsgChunksPerDay;
diagnosticInfo.OutgressBatchBlockBufferSize = _batchNetworkMessageBlock.OutputCount;
diagnosticInfo.OutgressInputBufferCount = _sinkBlock.InputCount;
diagnosticInfo.OutgressInputBufferDropped = _sinkBlockInputDroppedCount;
diagnosticInfo.OutgressIoTMessageCount = _messageSink.SentMessagesCount;
diagnosticInfo.ConnectionRetries = _messageTrigger.NumberOfConnectionRetries;
diagnosticInfo.OpcEndpointConnected = _messageTrigger.IsConnectionOk;
diagnosticInfo.MonitoredOpcNodesSucceededCount = _messageTrigger.NumberOfGoodNodes;
diagnosticInfo.MonitoredOpcNodesFailedCount = _messageTrigger.NumberOfBadNodes;
return(diagnosticInfo);
}
/// <inheritdoc/>
public async Task <HeartbeatResultModel> SendHeartbeatAsync(HeartbeatModel heartbeat,
JobDiagnosticInfoModel info, CancellationToken ct)
{
if (_workerRepository != null)
{
await _workerRepository.AddOrUpdate(heartbeat.Worker);
}
var result = new HeartbeatResultModel {
HeartbeatInstruction = HeartbeatInstruction.Keep,
LastActiveHeartbeat = null,
UpdatedJob = null
};
if (heartbeat.Job == null)
{
// Worker heartbeat
return(result);
}
var job = await _jobRepository.UpdateAsync(heartbeat.Job.JobId, existingJob => {
if (existingJob.GetHashSafe() != heartbeat.Job.JobHash)
{
// job was updated - instruct worker to reset
result.UpdatedJob = new JobProcessingInstructionModel {
Job = existingJob,
ProcessMode = heartbeat.Job.ProcessMode
};
}
if (existingJob.LifetimeData == null)
{
existingJob.LifetimeData = new JobLifetimeDataModel();
}
if (existingJob.LifetimeData.Status == JobStatus.Canceled ||
existingJob.LifetimeData.Status == JobStatus.Deleted)
{
result.HeartbeatInstruction = HeartbeatInstruction.CancelProcessing;
result.UpdatedJob = null;
result.LastActiveHeartbeat = null;
}
if (result.HeartbeatInstruction == HeartbeatInstruction.Keep)
{
existingJob.LifetimeData.Status = heartbeat.Job.Status;
}
var processingStatus = new ProcessingStatusModel {
LastKnownHeartbeat = DateTime.UtcNow,
LastKnownState = heartbeat.Job.State,
ProcessMode = // Unset processing mode to do correct calculation of active agents
heartbeat.Job.Status == JobStatus.Active ?
heartbeat.Job.ProcessMode : (ProcessMode?)null
};
if (existingJob.LifetimeData.ProcessingStatus == null)
{
existingJob.LifetimeData.ProcessingStatus = new Dictionary <string, ProcessingStatusModel>();
}
existingJob.LifetimeData.ProcessingStatus[heartbeat.Worker.WorkerId] = processingStatus;
var numberOfActiveAgents = existingJob.LifetimeData.ProcessingStatus
.Count(j =>
j.Value.ProcessMode == ProcessMode.Active &&
j.Value.LastKnownHeartbeat > DateTime.UtcNow.Subtract(_jobOrchestratorConfig.JobStaleTime));
if (processingStatus.ProcessMode == ProcessMode.Passive &&
numberOfActiveAgents < existingJob.RedundancyConfig.DesiredActiveAgents)
{
var lastActiveHeartbeat = existingJob.LifetimeData.ProcessingStatus
.Where(s => s.Value.ProcessMode == ProcessMode.Active)
.OrderByDescending(s => s.Value.LastKnownHeartbeat)
.Select(s => s.Value.LastKnownHeartbeat)
.FirstOrDefault();
// Switch this passive agent to active
result.HeartbeatInstruction = HeartbeatInstruction.SwitchToActive;
result.LastActiveHeartbeat = lastActiveHeartbeat;
existingJob.LifetimeData.ProcessingStatus[heartbeat.Worker.WorkerId].ProcessMode =
ProcessMode.Active;
}
return(Task.FromResult(true));
}, ct);
return(result);
}
