public async void ReplicateCreateOperation()
{
Guid recordId = Guid.NewGuid();
await _shardManager.Handle(new ReplicateShardWriteOperation()
{
Operation =
new ShardWriteOperation()
{
Id = Guid.NewGuid().ToString(),
Operation = ShardOperationOptions.Create,
Data = new TestData()
{
ShardId = TestUtility.DefaultShardId,
ShardType = "number",
Data = 100,
Id = recordId
},
Pos = 1,
ShardHash = ObjectUtility.HashStrings("", recordId.ToString())
}
});
var result = await _shardManager.Handle(
new RequestDataShard()
{
ObjectId = recordId,
Type = "number",
});
Assert.True(result.IsSuccessful);
Assert.NotNull(result.Data);
Assert.Equal(100, ((TestData)result.Data).Data);
}
public async void ConsecutiveReplicationTest()
{
Guid recordId = Guid.NewGuid();
string transactionId = Guid.NewGuid().ToString();
var firstTransaction = await _shardManager.Handle(new ReplicateShardWriteOperation()
{
Operation = new ShardWriteOperation()
{
Id = transactionId,
Operation = ShardOperationOptions.Create,
Pos = 1,
Data = new TestData()
{
ShardId = TestUtility.DefaultShardId,
ShardType = "number",
Data = 100,
Id = recordId
},
ShardHash = ObjectUtility.HashStrings("", transactionId)
}
});
var secondTransactionId = Guid.NewGuid().ToString();
await _shardManager.Handle(new ReplicateShardWriteOperation()
{
Operation = new ShardWriteOperation()
{
Id = secondTransactionId,
Operation = ShardOperationOptions.Update,
Pos = 2,
Data =
new TestData()
{
ShardId = TestUtility.DefaultShardId,
ShardType = "number",
Data = 101,
Id = recordId
},
ShardHash = ObjectUtility.HashStrings(ObjectUtility.HashStrings("", transactionId), secondTransactionId)
}
});
var result = await _shardManager.Handle(new RequestDataShard()
{
ObjectId = recordId,
Type = "number"
});
Assert.True(result.IsSuccessful);
Assert.NotNull(result.Data);
Assert.Equal(101, ((TestData)result.Data).Data);
}
public async Task <WriteShardDataResponse> WriteShardData(ShardData data, ShardOperationOptions operationType, string operationId, DateTime transactionDate)
{
ShardWriteOperation operation = new ShardWriteOperation()
{
Data = data,
Id = operationId,
Operation = operationType,
TransactionDate = transactionDate
};
ShardWriteOperation lastOperation = await GetOrPopulateOperationCache(operation.Data.ShardId.Value);
//Start at 1
operation.Pos = lastOperation == null ? 1 : lastOperation.Pos + 1;
var hash = lastOperation == null ? "" : lastOperation.ShardHash;
operation.ShardHash = ObjectUtility.HashStrings(hash, operation.Id);
_logger.LogDebug(_nodeStateService.GetNodeLogId() + "writing new operation " + operationId + " with data " + Environment.NewLine + JsonConvert.SerializeObject(data, Formatting.Indented));
//Write the data
var writeOperation = await _shardRepository.AddShardWriteOperationAsync(operation); //Add shard operation
if (writeOperation)
{
ApplyOperationToDatastore(operation);
var shardMetadata = _stateMachine.GetShard(operation.Data.ShardType, operation.Data.ShardId.Value);
//Mark operation as applied
await _shardRepository.MarkShardWriteOperationAppliedAsync(operation.Id);
//Update the cache
UpdateOperationCache(operation.Data.ShardId.Value, operation);
ConcurrentBag <Guid> InvalidNodes = new ConcurrentBag <Guid>();
//All allocations except for your own
var tasks = shardMetadata.InsyncAllocations.Where(id => id != _nodeStateService.Id).Select(async allocation =>
{
try
{
var result = await _clusterClient.Send(allocation, new ReplicateShardWriteOperation()
{
Operation = operation
});
if (result.IsSuccessful)
{
_logger.LogDebug(_nodeStateService.GetNodeLogId() + "Successfully replicated all " + shardMetadata.Id + "shards.");
}
else
{
throw new Exception("Failed to replicate data to shard " + shardMetadata.Id + " to node " + allocation + " for operation " + operation.ToString() + Environment.NewLine + JsonConvert.SerializeObject(operation, Formatting.Indented));
}
}
catch (TaskCanceledException e)
{
_logger.LogError(_nodeStateService.GetNodeLogId() + "Failed to replicate shard " + shardMetadata.Id + " on node " + allocation + " for operation " + operation.Pos + " as request timed out, marking shard as not insync...");
InvalidNodes.Add(allocation);
}
catch (Exception e)
{
_logger.LogError(_nodeStateService.GetNodeLogId() + "Failed to replicate shard " + shardMetadata.Id + " for operation " + operation.Pos + " with error " + e.Message + ", marking shard as not insync..." + Environment.NewLine + e.StackTrace);
InvalidNodes.Add(allocation);
}
});
await Task.WhenAll(tasks);
if (InvalidNodes.Count() > 0)
{
await _clusterClient.Send(new ExecuteCommands()
{
Commands = new List <BaseCommand>()
{
new UpdateShardMetadataAllocations()
{
ShardId = data.ShardId.Value,
Type = data.ShardType,
StaleAllocationsToAdd = InvalidNodes.ToHashSet(),
InsyncAllocationsToRemove = InvalidNodes.ToHashSet()
}
},
WaitForCommits = true
});
_logger.LogInformation(_nodeStateService.GetNodeLogId() + " had stale virtual machines.");
}
return(new WriteShardDataResponse()
{
Pos = operation.Pos,
ShardHash = operation.ShardHash,
IsSuccessful = true
});
}
else
{
return(new WriteShardDataResponse()
{
IsSuccessful = false
});
}
}
public async Task <bool> ReplicateShardWriteOperationAsync(ShardWriteOperation operation)
{
// Get the last operation
ShardWriteOperation lastOperation;
var startTime = DateTime.Now;
if (operation.Pos != 1)
{
while ((lastOperation = await GetOrPopulateOperationCache(operation.Data.ShardId.Value)) == null || !lastOperation.Applied || lastOperation.Pos < operation.Pos - 1)
{
//Assume in the next 3 seconds you should receive the previos requests
if ((DateTime.Now - startTime).TotalMilliseconds > 3000)
{
return(false);
}
await Task.Delay(100);
}
if (lastOperation.Pos != operation.Pos - 1)
{
lastOperation = await _shardRepository.GetShardWriteOperationAsync(operation.Data.ShardId.Value, operation.Pos - 1);
}
string newHash;
if ((newHash = ObjectUtility.HashStrings(lastOperation.ShardHash, operation.Id)) != operation.ShardHash)
{
//Critical error with data concurrency, revert back to last known good commit via the resync process
_logger.LogError(_nodeStateService.GetNodeLogId() + "Failed to check the hash for operation " + operation.Id + ", marking shard as out of sync...");
throw new Exception(_nodeStateService.GetNodeLogId() + "Failed to check the hash for operation " + operation.Id + ", marking shard as out of sync...");
}
}
// if (operation.Pos > 1)
//{
// lock (_shardLastOperationCache[operation.Data.ShardId.Value])
// {
//The operation must already be being applied
/* if (_shardLastOperationCache[operation.Data.ShardId.Value].Pos >= operation.Pos)
* {
* throw new Exception("There must be a replication process being run");
* }*/
_shardRepository.AddShardWriteOperationAsync(operation).GetAwaiter().GetResult();
//Run shard operation
ApplyOperationToDatastore(operation);
_shardRepository.MarkShardWriteOperationAppliedAsync(operation.Id).GetAwaiter().GetResult();
//Update the cache
UpdateOperationCache(operation.Data.ShardId.Value, operation);
//}
/* }
* else
* {
* await _shardRepository.AddShardWriteOperationAsync(operation);
* //Run shard operation
* ApplyOperationToDatastore(operation);
* await _shardRepository.MarkShardWriteOperationAppliedAsync(operation.Id);
* //Update the cache
* UpdateOperationCache(operation.Data.ShardId.Value, operation);
* }*/
return(true);
}