public async Task <ShardData> InsertDataAsync(ShardData data)
{
switch (data)
{
case TestData t1:
var addResult = _numberStore.TryAdd(t1.Id, t1);
if (!addResult)
{
// Console.WriteLine("Failed to insert data, there seems to be a concurrency issue! The data must already exist for object..." + t1.Id + " replacing the existing data" + Environment.NewLine + JsonConvert.SerializeObject(data, Formatting.Indented));
await UpdateDataAsync(data);
}
break;
}
return(data);
/*
* switch (data.Type)
* {
* case "number":
* lock (locker)
* {
* if (_numberStore.ContainsKey(data.Value))
* {
* _numberStore[assignedGuid.Value] = Convert.ToInt32(data);
* }
* else
* {
* _numberStore.Add(assignedGuid.Value, Convert.ToInt32(data));
* }
* }
* return assignedGuid.Value;
* }
* return assignedGuid.Value;*/
}
public async Task <ShardData> UpdateDataAsync(ShardData data)
{
switch (data)
{
case TestData t1:
if (!_numberStore.TryUpdate(data.Id, t1, _numberStore[data.Id]))
{
// Console.WriteLine("Failed to update data " + data.Id + " on shard " + data.ShardId + " due to concurrency issues");
throw new Exception("Failed to update data " + data.Id + " on shard " + data.ShardId + " due to concurrency issues");
}
else
{
// Console.WriteLine("Updated successfully object " + data.Id + " on shard " + data.ShardId + " due to concurrency issues");
}
/*if (!updateResult)
* {
* throw new Exception("Failed to update data " + data.Id + " on shard " + data.ShardId);
* }*/
break;
default:
throw new Exception("Failed to match the data with a type");
}
return(data);
}
public async Task <bool> DeleteDataAsync(ShardData data)
{
switch (data.ShardType)
{
case nameof(BotKey):
await _entitiesRepository.Delete <BotKey>(bk => bk.Id == data.Id);
break;
case nameof(ExecutionSchedule):
await _entitiesRepository.Delete <ExecutionSchedule>(bk => bk.Id == data.Id);
break;
case nameof(MetricTick):
await _entitiesRepository.Delete <MetricTick>(bk => bk.Id == data.Id);
break;
default:
throw new NotImplementedException();
}
return(true);
}
public async Task <ShardData> UpdateDataAsync(ShardData data)
{
data.ShardType = data.GetType().Name;
switch (data)
{
case User t1:
throw new Exception();
//return await _users.Upda(t1);
case BotKey t1:
return(await _entitiesRepository.Update(e => e.Id == data.Id, t1));
case GlobalValue t1:
return(await _entitiesRepository.Update(e => e.Id == data.Id, t1));
case Workflow t1:
return(await _entitiesRepository.Update(e => e.Id == data.Id, t1));
case Step t1:
return(await _entitiesRepository.Update(e => e.Id == data.Id, t1));
case ExecutionTemplate t1:
return(await _entitiesRepository.Update(e => e.Id == data.Id, t1));
case ExecutionSchedule t1:
return(await _entitiesRepository.Update(e => e.Id == data.Id, t1));
}
throw new Exception("Object type " + data.ShardType + " has no supported update operations");
}
public async void ReverseLocalTransaction(Guid shardId, string type, int pos)
{
var operation = await _shardRepository.GetShardWriteOperationAsync(shardId, pos);
if (operation != null)
{
await _shardRepository.AddDataReversionRecordAsync(new DataReversionRecord()
{
OriginalOperation = operation,
NewData = operation.Data,
OriginalData = await _dataRouter.GetDataAsync(type, operation.Data.Id),
RevertedTime = DateTime.Now
});
ShardData data = operation.Data;
SortedDictionary <int, ShardWriteOperation> allOperations;
ShardWriteOperation lastObjectOperation = null;
if (operation.Operation == ShardOperationOptions.Update || operation.Operation == ShardOperationOptions.Delete)
{
allOperations = await _shardRepository.GetAllObjectShardWriteOperationAsync(data.ShardId.Value, data.Id);
lastObjectOperation = allOperations.Where(ao => ao.Key < operation.Pos).Last().Value;
}
switch (operation.Operation)
{
case ShardOperationOptions.Create:
data = await _dataRouter.GetDataAsync(type, operation.Data.Id);
if (data != null)
{
_logger.LogInformation(_nodeStateService.GetNodeLogId() + "Reverting create with deletion of " + data);
await _dataRouter.DeleteDataAsync(data);
}
break;
// Put the data back in the right position
case ShardOperationOptions.Delete:
await _dataRouter.InsertDataAsync(lastObjectOperation.Data);
break;
case ShardOperationOptions.Update:
await _dataRouter.UpdateDataAsync(lastObjectOperation.Data);
break;
}
await _shardRepository.RemoveShardWriteOperationAsync(shardId, operation.Pos);
}
//Update the cache
_shardLastOperationCache.Remove(operation.Data.ShardId.Value, out _);
}
public async Task <WriteShardDataResponse> WriteShardData(ShardData data, ShardOperationOptions operationType, string operationId, DateTime transactionDate)
{
ShardWriteOperation operation = new ShardWriteOperation()
{
Data = data,
Id = operationId,
Operation = operationType,
TransactionDate = transactionDate
};
ShardWriteOperation lastOperation = await GetOrPopulateOperationCache(operation.Data.ShardId.Value);
//Start at 1
operation.Pos = lastOperation == null ? 1 : lastOperation.Pos + 1;
var hash = lastOperation == null ? "" : lastOperation.ShardHash;
operation.ShardHash = ObjectUtility.HashStrings(hash, operation.Id);
_logger.LogDebug(_nodeStateService.GetNodeLogId() + "writing new operation " + operationId + " with data " + Environment.NewLine + JsonConvert.SerializeObject(data, Formatting.Indented));
//Write the data
var writeOperation = await _shardRepository.AddShardWriteOperationAsync(operation); //Add shard operation
if (writeOperation)
{
ApplyOperationToDatastore(operation);
var shardMetadata = _stateMachine.GetShard(operation.Data.ShardType, operation.Data.ShardId.Value);
//Mark operation as applied
await _shardRepository.MarkShardWriteOperationAppliedAsync(operation.Id);
//Update the cache
UpdateOperationCache(operation.Data.ShardId.Value, operation);
ConcurrentBag <Guid> InvalidNodes = new ConcurrentBag <Guid>();
//All allocations except for your own
var tasks = shardMetadata.InsyncAllocations.Where(id => id != _nodeStateService.Id).Select(async allocation =>
{
try
{
var result = await _clusterClient.Send(allocation, new ReplicateShardWriteOperation()
{
Operation = operation
});
if (result.IsSuccessful)
{
_logger.LogDebug(_nodeStateService.GetNodeLogId() + "Successfully replicated all " + shardMetadata.Id + "shards.");
}
else
{
throw new Exception("Failed to replicate data to shard " + shardMetadata.Id + " to node " + allocation + " for operation " + operation.ToString() + Environment.NewLine + JsonConvert.SerializeObject(operation, Formatting.Indented));
}
}
catch (TaskCanceledException e)
{
_logger.LogError(_nodeStateService.GetNodeLogId() + "Failed to replicate shard " + shardMetadata.Id + " on node " + allocation + " for operation " + operation.Pos + " as request timed out, marking shard as not insync...");
InvalidNodes.Add(allocation);
}
catch (Exception e)
{
_logger.LogError(_nodeStateService.GetNodeLogId() + "Failed to replicate shard " + shardMetadata.Id + " for operation " + operation.Pos + " with error " + e.Message + ", marking shard as not insync..." + Environment.NewLine + e.StackTrace);
InvalidNodes.Add(allocation);
}
});
await Task.WhenAll(tasks);
if (InvalidNodes.Count() > 0)
{
await _clusterClient.Send(new ExecuteCommands()
{
Commands = new List <BaseCommand>()
{
new UpdateShardMetadataAllocations()
{
ShardId = data.ShardId.Value,
Type = data.ShardType,
StaleAllocationsToAdd = InvalidNodes.ToHashSet(),
InsyncAllocationsToRemove = InvalidNodes.ToHashSet()
}
},
WaitForCommits = true
});
_logger.LogInformation(_nodeStateService.GetNodeLogId() + " had stale virtual machines.");
}
return(new WriteShardDataResponse()
{
Pos = operation.Pos,
ShardHash = operation.ShardHash,
IsSuccessful = true
});
}
else
{
return(new WriteShardDataResponse()
{
IsSuccessful = false
});
}
}
public async Task <bool> DeleteDataAsync(ShardData data)
{
_numberStore.TryRemove(data.Id, out _);
return(true);
}
public async Task <ShardData> InsertDataAsync(ShardData data)
{
data.ShardType = data.GetType().Name;
try
{
switch (data)
{
case User t1:
return(await _entitiesRepository.Insert(t1));
case BotKey t1:
return(await _entitiesRepository.Insert(t1));
case GlobalValue t1:
return(await _entitiesRepository.Insert(t1));
case StepTemplate t1:
return(await _entitiesRepository.Insert(t1));
case WorkflowTemplate t1:
return(await _entitiesRepository.Insert(t1));
case Metric t1:
return(await _entitiesRepository.Insert(t1));
case MetricTick t1:
return(await _entitiesRepository.Insert(t1));
case Step t1:
return(await _entitiesRepository.Insert(t1));
case Workflow t1:
return(await _entitiesRepository.Insert(t1));
case ExecutionTemplate t1:
return(await _entitiesRepository.Insert(t1));
case ExecutionSchedule t1:
return(await _entitiesRepository.Insert(t1));
}
throw new Exception("Object type " + data.ShardType + "has no supported operations");
}
catch (Exception e)
{
if (e.Message.Contains("E11000 duplicate key error collection"))
{
Console.WriteLine("Detected that there was a duplicate record in the database... Updating existing record and continueing");
try
{
return(await UpdateDataAsync(data));
}
catch (Exception updateError)
{
if (updateError.Message.Contains("has no supported update operations"))
{
return(data);
}
else
{
throw updateError;
}
}
}
throw e;
}
}
public async Task <ShardData> GetData(Guid objectId, string type, int timeoutMs, Guid?shardId = null)
{
Guid?FoundShard = null;
Guid?FoundOnNode = null;
var currentTime = DateTime.Now;
if (shardId == null)
{
var shards = _stateMachine.GetAllPrimaryShards(type);
bool foundResult = false;
ShardData finalObject = null;
var totalRespondedShards = 0;
var tasks = shards.Select(async shard =>
{
if (shard.Value != _nodeStateService.Id)
{
try
{
var result = await _clusterClient.Send(shard.Value, new RequestDataShard()
{
ObjectId = objectId,
ShardId = shard.Key, //Set the shard
Type = type
});
if (result.IsSuccessful)
{
foundResult = true;
finalObject = result.Data;
FoundShard = result.ShardId;
FoundOnNode = result.NodeId;
}
Interlocked.Increment(ref totalRespondedShards);
}
catch (Exception e)
{
_logger.LogError(_nodeStateService.GetNodeLogId() + "Error thrown while getting " + e.Message);
}
}
else
{
finalObject = await _dataRouter.GetDataAsync(type, objectId);
foundResult = finalObject != null ? true : false;
FoundShard = shard.Key;
FoundShard = shard.Value;
Interlocked.Increment(ref totalRespondedShards);
}
});
//Don't await, this will trigger the tasks
Task.WhenAll(tasks);
while (!foundResult && totalRespondedShards < shards.Count)
{
if ((DateTime.Now - currentTime).TotalMilliseconds > timeoutMs)
{
throw new ClusterOperationTimeoutException("Get data request for object " + objectId + " from shard " + shardId + " timed out.");
}
await Task.Delay(10);
}
return(finalObject);
}
else
{
return(await _dataRouter.GetDataAsync(type, objectId));
}
}