public void CreateSnapshot(int indexIncludedTo, int trailingLogCount)
{
var stateMachine = new StateMachine <State>();
lock (_locker)
{
if (!LogExists(LastSnapshotIncludedIndex + 1))
{
return;
}
//Find the log position of the commit index
if (LastSnapshot != null)
{
stateMachine.ApplySnapshotToStateMachine(SystemExtension.Clone(LastSnapshot));
}
var logIncludedTo = GetLogAtIndex(indexIncludedTo);
var logIncludedFrom = GetLogAtIndex(LastSnapshotIncludedIndex + 1);
Logger.LogDebug("Last snapshot index, getting logs " + logIncludedFrom.Index + " to " + indexIncludedTo);
if (logIncludedTo != null && logIncludedFrom != null)
{
//for (var i = LastSnapshotIncludedIndex; i <= indexIncludedTo; i++)
stateMachine.ApplyLogsToStateMachine(GetLogRange(LastSnapshotIncludedIndex + 1, indexIncludedTo));
SetLastSnapshot(stateMachine.CurrentState, indexIncludedTo, logIncludedTo.Term);
// Deleting logs from the index
DeleteLogsFromIndex(1, indexIncludedTo - trailingLogCount < 0 ? 1 : indexIncludedTo - trailingLogCount);
}
}
}
public async Task <IActionResult> GetProgress()
{
var model = new List <TestsProgressModel>();
var allSuits = await _testLabProvider.GetAllTestSuits();
var finishedDate = DateTime.Parse("03/29/2019");
var testSuites = allSuits.ToList();
var tests = testSuites.SelectMany(p => p.TestCases).Where(p => p.MergedDate.HasValue && !p.IsIssue)
.GroupBy(p => p.MergedDate.Value.GetWeekOfYear()).OrderBy(p => p.Key);
var issues = testSuites.SelectMany(p => p.TestCases).Where(p => p.MergedDate.HasValue && p.IsIssue)
.GroupBy(p => p.MergedDate.Value.GetWeekOfYear()).OrderBy(p => p.Key);
var textJson = "[{\"label\": \"Tests\", \"data\": [";
foreach (var test in tests.Where(p => p.Key >= 13))
{
textJson += $@"[""{SystemExtension.FirstDateOfWeekISO8601(DateTime.Now.Year, test.Key):M}"", {test.Count()}], ";
}
textJson += "]";
textJson = textJson.Replace(", ]", "]},{\"label\":\"Issues\", \"data\":[");
foreach (var issue in issues.Where(p => p.Key >= 13))
{
textJson += $@"[""{SystemExtension.FirstDateOfWeekISO8601(DateTime.Now.Year, issue.Key):M}"", {issue.Count()}], ";
}
textJson = textJson.Substring(0, textJson.Length - 2) + "]}]";
return(Content(textJson, "application/json"));
}
public Task <bool> EnqueueOperationAsync(ShardWriteOperation data)
{
lock (queueLock)
{
OperationQueue.Add(SystemExtension.Clone(data));
}
return(Task.FromResult(true));
}
private async Task ScanTasks()
{
while (true)
{
if ((_nodeStateService.Role == NodeState.Follower || _nodeStateService.Role == NodeState.Leader) && _nodeStateService.IsBootstrapped && _nodeStateService.InCluster)
{
_logger.LogDebug(_nodeStateService.GetNodeLogId() + "Starting task watch.");
//Check tasks assigned to this node
var tasks = _stateMachine.CurrentState.GetNodeClusterTasks(new ClusterTaskStatuses[] { ClusterTaskStatuses.Created }, _nodeStateService.Id).ToList();
var currentTasksNo = _nodeTasks.Where(t => !t.Value.Task.IsCompleted).Count();
var numberOfTasksToAssign = (tasks.Count() > (_clusterOptions.ConcurrentTasks - currentTasksNo)) ? (_clusterOptions.ConcurrentTasks - currentTasksNo) : tasks.Count();
_logger.LogDebug(_nodeStateService.GetNodeLogId() + numberOfTasksToAssign + "tasks to run. || " + currentTasksNo);
if (numberOfTasksToAssign > 0)
{
await _clusterClient.Send(new ExecuteCommands()
{
Commands = new List <BaseCommand>()
{
new UpdateClusterTasks()
{
TasksToUpdate = tasks.GetRange(0, numberOfTasksToAssign).Select(t => new TaskUpdate()
{
Status = ClusterTaskStatuses.InProgress,
TaskId = t.Id
}).ToList()
}
},
WaitForCommits = true
});
//Create a thread for each task
for (var i = 0; i < numberOfTasksToAssign; i++)
{
_logger.LogDebug(_nodeStateService.GetNodeLogId() + " is starting task " + tasks[i].ToString());
try
{
var newTask = StartNodeTask(SystemExtension.Clone(tasks[i]));
_nodeTasks.TryAdd(tasks[i].Id
, new NodeTaskMetadata()
{
Id = tasks[i].Id,
Task = Task.Run(() => newTask)
});
}
catch (Exception e)
{
_logger.LogCritical(_nodeStateService.GetNodeLogId() + "Failed to fail step " + tasks[i].Id + " gracefully.");
}
}
}
}
await Task.Delay(1000);
}
}
// NOTE: Fisher–Yates Shuffle (algorithm)
public static void Shuffle <T>(this T[] array)
{
if (null == array)
{
return;
}
int count = array.Length;
for (int n = 0; n < count; ++n)
{
int indexToSwap = SystemExtension.RandomRange(n, count);
T oldValue = array[n];
array[n] = array[indexToSwap];
array[indexToSwap] = oldValue;
}
}
public async Task <ShardWriteOperation> DequeueOperation()
{
lock (queueLock)
{
var operationQueueResult = OperationQueue.Take(1);
if (operationQueueResult.Count() > 0)
{
var operation = operationQueueResult.First();
TransitQueue.TryAdd(operation.Id, SystemExtension.Clone(operation));
OperationQueue.Remove(operation);
if (_persistToDisk)
{
_operationCacheRepository.AddOperationToTransitAsync(operation).GetAwaiter().GetResult();
_operationCacheRepository.DeleteOperationFromQueueAsync(operation).GetAwaiter().GetResult();
}
return(operation);
}
return(null);
}
}
public Task <ShardWriteOperation> GetShardWriteOperationAsync(Guid shardId, int syncPos)
{
return(Task.FromResult(SystemExtension.Clone(ShardWriteOperations.Where(swo => swo.Value.Data.ShardId == shardId && swo.Value.Pos == syncPos).FirstOrDefault().Value)));
}
public Task <IEnumerable <ShardWriteOperation> > GetAllShardWriteOperationsAsync(Guid shardId)
{
return(Task.FromResult(SystemExtension.Clone(ShardWriteOperations.Select(so => so.Value))));
}
public Task <bool> AddShardWriteOperationAsync(ShardWriteOperation operation)
{
return(Task.FromResult(ShardWriteOperations.TryAdd(operation.Id, SystemExtension.Clone(operation))));
}
public Task <bool> AddDataReversionRecordAsync(DataReversionRecord record)
{
DataReversionRecords.Add(SystemExtension.Clone(record));
return(Task.FromResult(true));
}
public Task <bool> MarkObjectForDeletionAsync(ObjectDeletionMarker marker)
{
ObjectDeletionMarker.Add(SystemExtension.Clone(marker));
return(Task.FromResult(true));
}
public Task <bool> AddOperationToTransitAsync(ShardWriteOperation operation)
{
return(Task.FromResult(TransitQueue.TryAdd(operation.Id, SystemExtension.Clone(operation))));
}
public void ApplyCommand(BaseCommand command)
{
try
{
switch (command)
{
case UpsertNodeInformation t1:
if (Nodes.ContainsKey(t1.Id))
{
Nodes[t1.Id] = new NodeInformation()
{
Name = t1.Name,
TransportAddress = t1.TransportAddress,
Id = t1.Id,
IsContactable = t1.IsContactable
};
}
else
{
//Only add if it is marking as contactable
if (t1.IsContactable)
{
var nodes = Nodes.Where(n => n.Value.TransportAddress == t1.TransportAddress);
if (nodes.Count() > 0)
{
foreach (var node in nodes)
{
Nodes.Remove(node.Key, out _);
}
}
Nodes.TryAdd(t1.Id, new NodeInformation()
{
Name = t1.Name,
TransportAddress = t1.TransportAddress,
Id = t1.Id,
IsContactable = t1.IsContactable
});
}
}
break;
case DeleteNodeInformation t1:
if (Nodes.ContainsKey(t1.Id))
{
Nodes.TryRemove(t1.Id, out _);
}
break;
case CreateIndex t1:
Indexes.TryAdd(t1.Type, new Index()
{
Shards = t1.Shards.Select(s => s.DeepCopy()).ToList(),
Type = t1.Type
});
break;
case UpdateClusterTasks t1:
if (t1.TasksToAdd != null)
{
foreach (var task in t1.TasksToAdd)
{
if (GetRunningTask(task.UniqueRunningId) == null)
{
if (!ClusterTasks.TryAdd(task.Id, SystemExtension.Clone(task)))
{
//Can't add a task twice
if (ClusterTasks.ContainsKey(task.Id))
{
if (_logger == null)
{
Console.WriteLine("Critical error while trying to add cluster task " + task.Id + " the id already exists as the object " + JsonConvert.SerializeObject(ClusterTasks[task.Id], Formatting.Indented));
}
else
{
_logger.LogDebug("Critical error while trying to add cluster task " + task.Id + " the id already exists as the object " + JsonConvert.SerializeObject(ClusterTasks[task.Id], Formatting.Indented));
}
}
else
{
throw new Exception("Critical error while trying to add cluster task " + task.Id);
}
}
}
else
{
if (_logger == null)
{
Console.WriteLine("The task already exists and is running. Skipping addition of task " + task.Id);
}
else
{
_logger.LogInformation("The task already exists and is running. Skipping addition of task " + task.Id);
}
}
}
}
if (t1.TasksToRemove != null)
{
foreach (var task in t1.TasksToRemove)
{
if (!ClusterTasks.TryRemove(task, out _))
{
throw new Exception("Critical error while trying to remove cluster task " + task);
}
}
}
if (t1.TasksToUpdate != null)
{
foreach (var task in t1.TasksToUpdate)
{
if (ClusterTasks.ContainsKey(task.TaskId))
{
ClusterTasks[task.TaskId].CompletedOn = task.CompletedOn;
ClusterTasks[task.TaskId].Status = task.Status;
ClusterTasks[task.TaskId].ErrorMessage = task.ErrorMessage;
}
else
{
if (_logger == null)
{
Console.WriteLine("Critical error while trying to update cluster task " + task.TaskId + " task is not present in dictionary.");
}
else
{
_logger.LogInformation("Critical error while trying to update cluster task " + task.TaskId + " task is not present in dictionary.");
}
}
}
}
break;
case UpdateShardMetadataAllocations t1:
if (t1.InsyncAllocationsToAdd != null)
{
var newList = new HashSet <Guid>();
Indexes[t1.Type].Shards.Where(s => s.Id == t1.ShardId).FirstOrDefault().InsyncAllocations.ToList().ForEach(ia => newList.Add(ia));
foreach (var allocation in t1.InsyncAllocationsToAdd)
{
//Indexes[t1.Type].Shards.Where(s => s.Id == t1.ShardId).FirstOrDefault().InsyncAllocations = newList;
newList.Add(allocation);
}
Indexes[t1.Type].Shards.Where(s => s.Id == t1.ShardId).FirstOrDefault().InsyncAllocations = new HashSet <Guid>(newList);
// Indexes[t1.Type].Shards.Where(s => s.Id == t1.ShardId).FirstOrDefault().StaleAllocations = Indexes[t1.Type].Shards.Where(s => s.Id == t1.ShardId).FirstOrDefault().StaleAllocations.Where(sa => !t1.InsyncAllocationsToAdd.Contains(sa)).ToHashSet();
}
if (t1.InsyncAllocationsToRemove != null)
{
Indexes[t1.Type].Shards.Where(s => s.Id == t1.ShardId).FirstOrDefault().InsyncAllocations = new HashSet <Guid>(Indexes[t1.Type].Shards.Where(s => s.Id == t1.ShardId).FirstOrDefault().InsyncAllocations.Where(ia => !t1.InsyncAllocationsToRemove.Contains(ia)));
}
if (t1.StaleAllocationsToAdd != null)
{
var newList = new HashSet <Guid>();
Indexes[t1.Type].Shards.Where(s => s.Id == t1.ShardId).FirstOrDefault().StaleAllocations.ToList().ForEach(ia => newList.Add(ia));
foreach (var allocation in t1.StaleAllocationsToAdd)
{
newList.Add(allocation);
}
Indexes[t1.Type].Shards.Where(s => s.Id == t1.ShardId).FirstOrDefault().StaleAllocations = new HashSet <Guid>(newList);
}
if (t1.StaleAllocationsToRemove != null)
{
var newList = new HashSet <Guid>();
Indexes[t1.Type].Shards.Where(s => s.Id == t1.ShardId).FirstOrDefault().StaleAllocations.Where(sa => !t1.StaleAllocationsToRemove.Contains(sa)).ToList().ForEach(ia => newList.Add(ia));
Indexes[t1.Type].Shards.Where(s => s.Id == t1.ShardId).FirstOrDefault().StaleAllocations = new HashSet <Guid>(newList);
}
if (t1.PrimaryAllocation != null)
{
Indexes[t1.Type].Shards.Where(s => s.Id == t1.ShardId).FirstOrDefault().PrimaryAllocation = t1.PrimaryAllocation.Value;
}
break;
case SetLock t1:
var stringLockResult = Locks.TryAdd(t1.Name, new Lock()
{
LockTimeoutMs = t1.TimeoutMs,
Name = t1.Name,
LockId = t1.LockId,
CreatedOn = t1.CreatedOn
});
if (!stringLockResult)
{
throw new ConflictingObjectLockException("String " + t1.Name + " is already locked.");
}
break;
case RemoveLock t1:
string lockName = t1.Name;
var objectLocksWithLockId = Locks.Where(ob => ob.Value.LockId == t1.LockId).Select(t => t.Value);
if (objectLocksWithLockId.Count() == 0)
{
throw new ConflictingObjectLockException("No lock with id " + t1.LockId);
}
Lock existingStringLock;
var removeStringLockResult = Locks.TryRemove(t1.Name, out existingStringLock);
if (!removeStringLockResult)
{
throw new ConflictingObjectLockException("String Lock " + t1.Name + " did not exist in locks.");
}
break;
default:
ApplyCommandToState(command);
break;
}
}
catch (Exception e)
{
throw new Exception("Failed to apply command " + command.CommandName + " to state with exception \"" + e.Message + "\"." + Environment.NewLine + JsonConvert.SerializeObject(command, Formatting.Indented));
}
}
public IEnumerable <BaseTask> GetNodeClusterTasks(ClusterTaskStatuses[] statuses, Guid nodeId)
{
return(SystemExtension.Clone(ClusterTasks.Where(ct => statuses.Contains(ct.Value.Status) && ct.Value.NodeId == nodeId).Select(b => b.Value)));
}