public bool TryAddNewExceptionInfo([NotNull] TaskMetaInformation taskMeta, [NotNull] Exception exception, out List <TimeGuid> newExceptionInfoIds)
{
if (!taskMeta.IsTimeBased())
{
throw new InvalidOperationException(string.Format("TaskMeta is not time-based: {0}", taskMeta));
}
newExceptionInfoIds = null;
var newExceptionInfo = new TaskExceptionInfo(exception);
var lastExceptionInfo = TryGetLastExceptionInfo(taskMeta);
if (lastExceptionInfo != null && lastExceptionInfo.ExceptionMessageInfo == newExceptionInfo.ExceptionMessageInfo)
{
return(false);
}
var newExceptionInfoId = TimeGuid.NowGuid();
var timestamp = newExceptionInfoId.GetTimestamp().Ticks;
TimeGuid oldExceptionInfoId;
newExceptionInfoIds = taskMeta.AddExceptionInfoId(newExceptionInfoId, out oldExceptionInfoId);
var newExceptionInfoBytes = serializer.Serialize(newExceptionInfo);
timeBasedBlobStorage.Write(taskMeta.Id, newExceptionInfoId, newExceptionInfoBytes, timestamp, taskMeta.GetTtl());
if (oldExceptionInfoId != null)
{
timeBasedBlobStorage.Delete(taskMeta.Id, oldExceptionInfoId, timestamp);
}
return(true);
}
private TaskMetaInformation TrySwitchToWaitingForRerunState([NotNull] TaskMetaInformation inProcessMeta, TaskState waitingForRerunState, TimeSpan rerunDelay, [CanBeNull] List <TimeGuid> newExceptionInfoIds)
{
var nowTicks = Timestamp.Now.Ticks;
var inProcessTaskIndexRecord = handleTasksMetaStorage.FormatIndexRecord(inProcessMeta);
return(TryUpdateTaskState(inProcessMeta, inProcessTaskIndexRecord, nowTicks + rerunDelay.Ticks, inProcessMeta.StartExecutingTicks, nowTicks, inProcessMeta.Attempts, waitingForRerunState, newExceptionInfoIds));
}
public void TaskWithSameIdsTest()
{
var ticks = Timestamp.Now.Ticks;
var id = NewTaskId();
var taskMeta1 = new TaskMetaInformation("TaskName", id)
{
State = TaskState.New, MinimalStartTicks = ticks + 10
};
sut.AddMeta(taskMeta1, oldTaskIndexRecord: null);
var taskMeta2 = new TaskMetaInformation("TaskName", id)
{
State = TaskState.InProcess, MinimalStartTicks = ticks + 15
};
sut.AddMeta(taskMeta2, oldTaskIndexRecord: null);
var newTasks = sut.GetIndexRecords(ticks + 12, new[] { TaskIndexShardKey("TaskName", TaskState.New) });
Assert.AreEqual(1, newTasks.Length);
Assert.AreEqual(id, newTasks[0].TaskId);
var inProcessTasks = sut.GetIndexRecords(ticks + 12, new[] { TaskIndexShardKey("TaskName", TaskState.InProcess) });
Assert.AreEqual(0, inProcessTasks.Length);
inProcessTasks = sut.GetIndexRecords(ticks + 16, new[] { TaskIndexShardKey("TaskName", TaskState.InProcess) });
Assert.AreEqual(1, inProcessTasks.Length);
Assert.AreEqual(id, inProcessTasks[0].TaskId);
}
public void Write([NotNull] TaskMetaInformation taskMeta, long timestamp)
{
var blobId = GetBlobId(taskMeta.Id);
var taskMetaBytes = serializer.Serialize(taskMeta);
timeBasedBlobStorage.Write(blobId, taskMetaBytes, timestamp, taskMeta.GetTtl());
}
public RemoteTaskInitialTraceContext([NotNull] TaskMetaInformation taskMeta)
{
/*traceContext = Trace.CreateChildContext(taskMeta.Name, taskMeta.Id);
* taskMeta.TraceId = traceContext.TraceId;
* taskMeta.TraceIsActive = traceContext.IsActive;
* traceContext.RecordTimepoint(Timepoint.Start, new DateTime(taskMeta.Ticks, DateTimeKind.Utc));*/
}
public TaskIndexRecord FormatIndexRecord([NotNull] TaskMetaInformation taskMeta)
{
var taskTopic = taskDataRegistry.GetTaskTopic(taskMeta.Name);
var taskIndexShardKey = new TaskIndexShardKey(taskTopic, taskMeta.State);
return(new TaskIndexRecord(taskMeta.Id, taskMeta.MinimalStartTicks, taskIndexShardKey));
}
public TaskIndexRecord AddMeta([NotNull] TaskMetaInformation taskMeta, [CanBeNull] TaskIndexRecord oldTaskIndexRecord)
{
var metricsContext = MetricsContext.For(taskMeta).SubContext("HandleTasksMetaStorage.AddMeta");
var globalNowTicks = globalTime.UpdateNowTimestamp().Ticks;
var nowTicks = Math.Max((taskMeta.LastModificationTicks ?? 0) + PreciseTimestampGenerator.TicksPerMicrosecond, globalNowTicks);
taskMeta.LastModificationTicks = nowTicks;
using (metricsContext.Timer("EventLogRepository_AddEvent").NewContext())
eventLogRepository.AddEvent(taskMeta, eventTimestamp: new Timestamp(nowTicks), eventId: Guid.NewGuid());
var newIndexRecord = FormatIndexRecord(taskMeta);
using (metricsContext.Timer("MinimalStartTicksIndex_AddRecord").NewContext())
minimalStartTicksIndex.AddRecord(newIndexRecord, globalNowTicks, taskMeta.GetTtl());
if (taskMeta.State == TaskState.New)
{
using (metricsContext.Timer("ChildTaskIndex_WriteIndexRecord").NewContext())
childTaskIndex.WriteIndexRecord(taskMeta, globalNowTicks);
}
using (metricsContext.Timer("TaskMetaStorage_Write").NewContext())
taskMetaStorage.Write(taskMeta, globalNowTicks);
if (oldTaskIndexRecord != null)
{
using (metricsContext.Timer("MinimalStartTicksIndex_RemoveRecord").NewContext())
minimalStartTicksIndex.RemoveRecord(oldTaskIndexRecord, globalNowTicks);
}
return(newIndexRecord);
}
private TaskMetaInformation TryUpdateTaskState([NotNull] TaskMetaInformation oldMeta, [NotNull] TaskIndexRecord oldTaskIndexRecord, long newMinimalStartTicks, long?startExecutingTicks, long?finishExecutingTicks, int attempts, TaskState newState, [CanBeNull] List <TimeGuid> newExceptionInfoIds)
{
var newMeta = allFieldsSerializer.Copy(oldMeta);
if (newState == oldMeta.State)
{
newMinimalStartTicks = Math.Max(newMinimalStartTicks, oldMeta.MinimalStartTicks + PreciseTimestampGenerator.TicksPerMicrosecond);
}
newMeta.MinimalStartTicks = newMinimalStartTicks;
newMeta.StartExecutingTicks = startExecutingTicks;
newMeta.FinishExecutingTicks = finishExecutingTicks;
newMeta.Attempts = attempts;
newMeta.State = newState;
if (newExceptionInfoIds != null && newExceptionInfoIds.Any())
{
newMeta.TaskExceptionInfoIds = newExceptionInfoIds;
}
try
{
handleTasksMetaStorage.AddMeta(newMeta, oldTaskIndexRecord);
logger.Debug("Changed task state. NewMeta: {RtqTaskMeta}", new { RtqTaskMeta = newMeta });
return(newMeta);
}
catch (Exception e)
{
logger.Error(e, "Can't update task state. OldMeta: {RtqTaskMeta}", new { RtqTaskMeta = oldMeta });
return(null);
}
}
private TaskMetaInformation TrySwitchToInProcessState([NotNull] TaskMetaInformation oldMeta)
{
var nowTicks = Timestamp.Now.Ticks;
var newMinimalStartTicks = nowTicks + CassandraNameHelper.TaskMinimalStartTicksIndexTicksPartition;
return(TryUpdateTaskState(oldMeta, taskIndexRecord, newMinimalStartTicks, nowTicks, null, oldMeta.Attempts + 1, TaskState.InProcess, newExceptionInfoIds: null));
}
public void ProlongMetaTtl([NotNull] TaskMetaInformation taskMeta)
{
var globalNowTicks = globalTime.UpdateNowTimestamp().Ticks;
minimalStartTicksIndex.WriteRecord(FormatIndexRecord(taskMeta), globalNowTicks, taskMeta.GetTtl());
childTaskIndex.WriteIndexRecord(taskMeta, globalNowTicks);
taskMetaStorage.Write(taskMeta, globalNowTicks);
}
private void WriteEvents(params EventWithOffset <TaskMetaUpdatedEvent, string>[] events)
{
foreach (var eventWithOffset in events)
{
var taskMeta = new TaskMetaInformation("taskName", eventWithOffset.Event.TaskId);
eventLogRepository.AddEvent(taskMeta, GetTimestamp(eventWithOffset), GetEventId(eventWithOffset));
}
}
public byte[] Read([NotNull] TaskMetaInformation taskMeta)
{
if (!taskMeta.IsTimeBased())
{
throw new InvalidOperationException(string.Format("TaskMeta is not time-based: {0}", taskMeta));
}
return(timeBasedBlobStorage.Read(taskMeta.GetTaskDataId()));
}
public void Overwrite([NotNull] TaskMetaInformation taskMeta, [NotNull] byte[] taskData)
{
if (!taskMeta.IsTimeBased())
{
throw new InvalidOperationException(string.Format("TaskMeta is not time-based: {0}", taskMeta));
}
timeBasedBlobStorage.Write(taskMeta.GetTaskDataId(), taskData, timestamp: Timestamp.Now.Ticks, ttl: taskMeta.GetTtl());
}
public RemoteTaskHandlingTraceContext([CanBeNull] TaskMetaInformation taskMeta)
{
TaskIsBeingTraced = taskMeta != null;
/*if (taskMeta != null)
* {
* traceContext = Trace.ContinueContext(taskMeta.TraceId, taskMeta.Id, taskMeta.TraceIsActive, isRoot : true);
* traceContext.RecordTimepoint(Timepoint.Start, new DateTime(taskMeta.Ticks, DateTimeKind.Utc));
* }*/
}
public void SetOrUpdateTtl_Now()
{
var now = Timestamp.Now;
var meta = new TaskMetaInformation("Test_name", "Test-id");
var ttl = TimeSpan.FromMilliseconds(3342);
meta.SetOrUpdateTtl(ttl);
Assert.That(meta.GetTtl(), Is.InRange(ttl - TimeSpan.FromSeconds(1), ttl));
Assert.That(meta.ExpirationTimestampTicks, Is.GreaterThanOrEqualTo((now + ttl).Ticks));
}
private static TaskMetaInformation TaskMeta(string taskId, TimeSpan?ttl)
{
var taskMeta = new TaskMetaInformation($"Name-{Guid.NewGuid():N}", taskId)
{
MinimalStartTicks = Timestamp.Now.Ticks
};
taskMeta.SetOrUpdateTtl(ttl ?? defaultTtl);
return(taskMeta);
}
private void Write(string taskId, TimeSpan?ttl = null)
{
var now = Timestamp.Now;
var taskMeta = new TaskMetaInformation("TaskName", taskId)
{
MinimalStartTicks = now.Ticks
};
taskMeta.SetOrUpdateTtl(ttl ?? defaultTtl);
taskMetaStorage.Write(taskMeta, now.Ticks);
}
public BlobId Write([NotNull] TaskMetaInformation taskMeta, [NotNull] byte[] taskData)
{
if (!taskMeta.IsTimeBased())
{
throw new InvalidOperationException(string.Format("TaskId is not time-based: {0}", taskMeta.Id));
}
var blobId = TimeBasedBlobStorage.GenerateNewBlobId(taskData.Length);
var timestamp = blobId.Id.GetTimestamp().Ticks;
timeBasedBlobStorage.Write(blobId, taskData, timestamp, taskMeta.GetTtl());
return(blobId);
}
public void AddEvent([NotNull] TaskMetaInformation taskMeta, [NotNull] Timestamp eventTimestamp, Guid eventId)
{
minTicksHolder.UpdateMinTicks(firstEventTicksRowName, eventTimestamp.Ticks);
var ttl = taskMeta.GetTtl();
cfConnection.AddColumn(EventPointerFormatter.GetPartitionKey(eventTimestamp.Ticks), new Column
{
Name = EventPointerFormatter.GetColumnName(eventTimestamp.Ticks, eventId),
Value = serializer.Serialize(new TaskMetaUpdatedEvent(taskMeta.Id, eventTimestamp.Ticks)),
Timestamp = eventTimestamp.Ticks,
TTL = ttl.HasValue ? (int)ttl.Value.TotalSeconds : (int?)null,
});
}
public void SetOrUpdateTtl_MinimalStartTicks()
{
var minimalStart = Timestamp.Now + TimeSpan.FromHours(1);
var ttl = TimeSpan.FromMilliseconds(3342);
var meta = new TaskMetaInformation("Test_name", "Test-id")
{
MinimalStartTicks = minimalStart.Ticks
};
meta.SetOrUpdateTtl(ttl);
Assert.That(meta.GetTtl(), Is.InRange(TimeSpan.FromHours(1) - ttl, TimeSpan.FromHours(1) + ttl));
Assert.That(meta.ExpirationTimestampTicks, Is.GreaterThanOrEqualTo((minimalStart + ttl).Ticks));
}
public void Delete([NotNull] TaskMetaInformation taskMeta)
{
if (!taskMeta.IsTimeBased())
{
throw new InvalidOperationException(string.Format("TaskMeta is not time-based: {0}", taskMeta));
}
var timestamp = Timestamp.Now.Ticks;
foreach (var blobId in taskMeta.GetTaskExceptionInfoIds())
{
timeBasedBlobStorage.Delete(taskMeta.Id, blobId, timestamp);
}
}
public void ProlongExceptionInfosTtl([NotNull] TaskMetaInformation taskMeta)
{
if (!taskMeta.IsTimeBased())
{
throw new InvalidOperationException(string.Format("TaskMeta is not time-based: {0}", taskMeta));
}
var oldExceptionInfos = timeBasedBlobStorage.Read(taskMeta.Id, taskMeta.GetTaskExceptionInfoIds().ToArray());
var timestamp = Timestamp.Now.Ticks;
foreach (var exceptionInfo in oldExceptionInfos)
{
timeBasedBlobStorage.Write(taskMeta.Id, exceptionInfo.Key, exceptionInfo.Value, timestamp, taskMeta.GetTtl());
}
}
private List <TimeGuid> TrуAddExceptionInfo([NotNull] Exception e, [NotNull] TaskMetaInformation inProcessMeta)
{
try
{
if (taskExceptionInfoStorage.TryAddNewExceptionInfo(inProcessMeta, e, out var newExceptionInfoIds))
{
return(newExceptionInfoIds);
}
}
catch (Exception exception)
{
logger.Error(exception, "Cannot add exception info for: {RtqTaskMeta}. Error: {Error}", new { RtqTaskMeta = inProcessMeta, Error = e });
}
return(null);
}
public void UpdateTaskState([NotNull] TaskMetaInformation taskMetaInformation)
{
lock (state)
{
if (!state.TaskMetas.TryGetValue(taskMetaInformation.Id, out var taskMeta))
{
taskMeta = new RtqTaskCounterStateTaskMeta(taskMetaInformation.Name);
state.TaskMetas.Add(taskMetaInformation.Id, taskMeta);
}
taskMeta.State = taskMetaInformation.State;
taskMeta.MinimalStartTimestamp = taskMetaInformation.GetMinimalStartTimestamp();
taskMeta.LastModificationTicks = taskMetaInformation.LastModificationTicks;
taskMeta.LastStateUpdateTimestamp = Timestamp.Now;
}
}
public void StressTest()
{
var nowTicks = Timestamp.Now.Ticks;
var meta = new TaskMetaInformation("TaskName", NewTaskId())
{
State = TaskState.New, MinimalStartTicks = nowTicks + 1
};
for (var i = 0; i <= 1000; i++)
{
var oldTaskIndexRecord = sut.FormatIndexRecord(meta);
meta.MinimalStartTicks++;
sut.AddMeta(meta, oldTaskIndexRecord);
}
Assert.AreEqual(1, sut.GetIndexRecords(nowTicks + 1002, new[] { TaskIndexShardKey("TaskName", TaskState.New) }).Length);
}
public void WriteIndexRecord([NotNull] TaskMetaInformation taskMeta, long timestamp)
{
if (string.IsNullOrEmpty(taskMeta.ParentTaskId))
{
return;
}
var ttl = taskMeta.GetTtl();
cassandraCluster.RetrieveColumnFamilyConnection(rtqSettings.QueueKeyspace, ColumnFamilyName).AddColumn(taskMeta.ParentTaskId, new Column
{
Name = taskMeta.Id,
Timestamp = timestamp,
Value = serializer.Serialize(taskMeta.Id),
TTL = ttl.HasValue ? (int)ttl.Value.TotalSeconds : (int?)null,
});
}
private ProcessTaskResult DoProcessTask([NotNull] TaskMetaInformation inProcessMeta, [NotNull] byte[] taskData, [NotNull] MetricsContext metricsContext)
{
metricsContext = metricsContext.SubContext(nameof(DoProcessTask));
using (metricsContext.Timer("Total").NewContext())
{
IRtqTaskHandler taskHandler;
try
{
using (metricsContext.Timer("CreateHandlerFor").NewContext())
taskHandler = taskHandlerRegistry.CreateHandlerFor(inProcessMeta.Name);
}
catch (Exception e)
{
logger.Error(e, "TaskHandler creation error for: {RtqTaskMeta}", new { RtqTaskMeta = inProcessMeta });
var newExceptionInfoIds = TrуAddExceptionInfo(e, inProcessMeta);
using (metricsContext.Timer("TrySwitchToTerminalState").NewContext())
return(new ProcessTaskResult(LocalTaskProcessingResult.Error, TrySwitchToTerminalState(inProcessMeta, TaskState.Fatal, newExceptionInfoIds)));
}
var task = new Task(inProcessMeta, taskData);
using (TaskExecutionContext.ForTask(task))
{
var sw = Stopwatch.StartNew();
try
{
HandleResult handleResult;
using (metricsContext.Timer("HandleTask").NewContext())
handleResult = taskHandler.HandleTask(taskProducer, serializer, task);
rtqProfiler.ProcessTaskExecutionFinished(inProcessMeta, handleResult, sw.Elapsed);
MetricsContext.For(inProcessMeta).Meter("TasksExecuted").Mark();
using (metricsContext.Timer("UpdateTaskMetaByHandleResult").NewContext())
return(UpdateTaskMetaByHandleResult(inProcessMeta, handleResult));
}
catch (Exception e)
{
rtqProfiler.ProcessTaskExecutionFailed(inProcessMeta, sw.Elapsed);
MetricsContext.For(inProcessMeta).Meter("TasksExecutionFailed").Mark();
logger.Error(e, "Unhandled exception in TaskHandler for: {RtqTaskMeta}", new { RtqTaskMeta = inProcessMeta });
var taskExceptionInfoId = TrуAddExceptionInfo(e, inProcessMeta);
using (metricsContext.Timer("TrySwitchToTerminalState").NewContext())
return(new ProcessTaskResult(LocalTaskProcessingResult.Error, TrySwitchToTerminalState(inProcessMeta, TaskState.Fatal, taskExceptionInfoId)));
}
}
}
}
public void DifferentNamesTest()
{
var ticks = Timestamp.Now.Ticks;
var taskMeta1 = new TaskMetaInformation("TaskName1", NewTaskId())
{
State = TaskState.New, MinimalStartTicks = ticks
};
sut.AddMeta(taskMeta1, oldTaskIndexRecord: null);
var taskMeta2 = new TaskMetaInformation("TaskName2", NewTaskId())
{
State = TaskState.New, MinimalStartTicks = ticks
};
sut.AddMeta(taskMeta2, oldTaskIndexRecord: null);
var tasks = sut.GetIndexRecords(ticks + 1, new[] { TaskIndexShardKey("TaskName1", TaskState.New) });
Assert.AreEqual(1, tasks.Length);
}
public static RtqMonitoringTaskMeta ToMonitoringTaskMeta(this TaskMetaInformation meta)
{
return(new RtqMonitoringTaskMeta
{
Name = meta.Name,
Id = meta.Id,
Ticks = meta.Ticks,
MinimalStartTicks = meta.MinimalStartTicks,
StartExecutingTicks = meta.StartExecutingTicks,
FinishExecutingTicks = meta.FinishExecutingTicks,
LastModificationTicks = meta.LastModificationTicks,
ExpirationTimestampTicks = meta.ExpirationTimestampTicks,
ExpirationModificationTicks = meta.ExpirationModificationTicks,
ExecutionDurationTicks = meta.ExecutionDurationTicks,
State = meta.State,
Attempts = meta.Attempts,
ParentTaskId = meta.ParentTaskId,
});
}
private TaskExceptionInfo TryGetLastExceptionInfo([NotNull] TaskMetaInformation taskMeta)
{
if (!taskMeta.IsTimeBased())
{
throw new InvalidOperationException(string.Format("TaskMeta is not time-based: {0}", taskMeta));
}
var lastExceptionInfoId = taskMeta.GetTaskExceptionInfoIds().LastOrDefault();
if (lastExceptionInfoId == null)
{
return(null);
}
var lastExceptionInfoBytes = timeBasedBlobStorage.Read(taskMeta.Id, lastExceptionInfoId);
if (lastExceptionInfoBytes == null)
{
return(null);
}
return(serializer.Deserialize <TaskExceptionInfo>(lastExceptionInfoBytes));
}