public Task <bool> EnqueueOperationAsync(ShardWriteOperation data) { lock (queueLock) { OperationQueue.Add(SystemExtension.Clone(data)); } return(Task.FromResult(true)); }
public Task <bool> DeleteOperationFromQueueAsync(ShardWriteOperation operation) { lock (queueLock) { OperationQueue.Remove(operation); } return(Task.FromResult(true)); }
public async void ReverseLocalTransaction(Guid shardId, string type, int pos) { var operation = await _shardRepository.GetShardWriteOperationAsync(shardId, pos); if (operation != null) { await _shardRepository.AddDataReversionRecordAsync(new DataReversionRecord() { OriginalOperation = operation, NewData = operation.Data, OriginalData = await _dataRouter.GetDataAsync(type, operation.Data.Id), RevertedTime = DateTime.Now }); ShardData data = operation.Data; SortedDictionary <int, ShardWriteOperation> allOperations; ShardWriteOperation lastObjectOperation = null; if (operation.Operation == ShardOperationOptions.Update || operation.Operation == ShardOperationOptions.Delete) { allOperations = await _shardRepository.GetAllObjectShardWriteOperationAsync(data.ShardId.Value, data.Id); lastObjectOperation = allOperations.Where(ao => ao.Key < operation.Pos).Last().Value; } switch (operation.Operation) { case ShardOperationOptions.Create: data = await _dataRouter.GetDataAsync(type, operation.Data.Id); if (data != null) { _logger.LogInformation(_nodeStateService.GetNodeLogId() + "Reverting create with deletion of " + data); await _dataRouter.DeleteDataAsync(data); } break; // Put the data back in the right position case ShardOperationOptions.Delete: await _dataRouter.InsertDataAsync(lastObjectOperation.Data); break; case ShardOperationOptions.Update: await _dataRouter.UpdateDataAsync(lastObjectOperation.Data); break; } await _shardRepository.RemoveShardWriteOperationAsync(shardId, operation.Pos); } //Update the cache _shardLastOperationCache.Remove(operation.Data.ShardId.Value, out _); }
public async Task <bool> EnqueueOperationAsync(ShardWriteOperation transaction) { lock (queueLock) { OperationQueue.Add(transaction); } if (_persistToDisk) { return(await _operationCacheRepository.EnqueueOperationAsync(transaction)); } return(true); }
public async Task <bool> AddShardWriteOperationAsync(ShardWriteOperation operation) { try { await _shardWriteOperations.InsertOneAsync(operation); } //Return true if the operation already exists catch (MongoDuplicateKeyException e) { return(true); } return(true); }
public async void ApplyOperationToDatastore(ShardWriteOperation operation) { switch (operation.Operation) { case ShardOperationOptions.Create: await _dataRouter.InsertDataAsync(operation.Data); break; case ShardOperationOptions.Delete: await _dataRouter.DeleteDataAsync(operation.Data); break; case ShardOperationOptions.Update: await _dataRouter.UpdateDataAsync(operation.Data); break; } }
public async Task <WriteShardDataResponse> WriteShardData(ShardData data, ShardOperationOptions operationType, string operationId, DateTime transactionDate) { ShardWriteOperation operation = new ShardWriteOperation() { Data = data, Id = operationId, Operation = operationType, TransactionDate = transactionDate }; ShardWriteOperation lastOperation = await GetOrPopulateOperationCache(operation.Data.ShardId.Value); //Start at 1 operation.Pos = lastOperation == null ? 1 : lastOperation.Pos + 1; var hash = lastOperation == null ? "" : lastOperation.ShardHash; operation.ShardHash = ObjectUtility.HashStrings(hash, operation.Id); _logger.LogDebug(_nodeStateService.GetNodeLogId() + "writing new operation " + operationId + " with data " + Environment.NewLine + JsonConvert.SerializeObject(data, Formatting.Indented)); //Write the data var writeOperation = await _shardRepository.AddShardWriteOperationAsync(operation); //Add shard operation if (writeOperation) { ApplyOperationToDatastore(operation); var shardMetadata = _stateMachine.GetShard(operation.Data.ShardType, operation.Data.ShardId.Value); //Mark operation as applied await _shardRepository.MarkShardWriteOperationAppliedAsync(operation.Id); //Update the cache UpdateOperationCache(operation.Data.ShardId.Value, operation); ConcurrentBag <Guid> InvalidNodes = new ConcurrentBag <Guid>(); //All allocations except for your own var tasks = shardMetadata.InsyncAllocations.Where(id => id != _nodeStateService.Id).Select(async allocation => { try { var result = await _clusterClient.Send(allocation, new ReplicateShardWriteOperation() { Operation = operation }); if (result.IsSuccessful) { _logger.LogDebug(_nodeStateService.GetNodeLogId() + "Successfully replicated all " + shardMetadata.Id + "shards."); } else { throw new Exception("Failed to replicate data to shard " + shardMetadata.Id + " to node " + allocation + " for operation " + operation.ToString() + Environment.NewLine + JsonConvert.SerializeObject(operation, Formatting.Indented)); } } catch (TaskCanceledException e) { _logger.LogError(_nodeStateService.GetNodeLogId() + "Failed to replicate shard " + shardMetadata.Id + " on node " + allocation + " for operation " + operation.Pos + " as request timed out, marking shard as not insync..."); InvalidNodes.Add(allocation); } catch (Exception e) { _logger.LogError(_nodeStateService.GetNodeLogId() + "Failed to replicate shard " + shardMetadata.Id + " for operation " + operation.Pos + " with error " + e.Message + ", marking shard as not insync..." + Environment.NewLine + e.StackTrace); InvalidNodes.Add(allocation); } }); await Task.WhenAll(tasks); if (InvalidNodes.Count() > 0) { await _clusterClient.Send(new ExecuteCommands() { Commands = new List <BaseCommand>() { new UpdateShardMetadataAllocations() { ShardId = data.ShardId.Value, Type = data.ShardType, StaleAllocationsToAdd = InvalidNodes.ToHashSet(), InsyncAllocationsToRemove = InvalidNodes.ToHashSet() } }, WaitForCommits = true }); _logger.LogInformation(_nodeStateService.GetNodeLogId() + " had stale virtual machines."); } return(new WriteShardDataResponse() { Pos = operation.Pos, ShardHash = operation.ShardHash, IsSuccessful = true }); } else { return(new WriteShardDataResponse() { IsSuccessful = false }); } }
private void UpdateOperationCache(Guid shardId, ShardWriteOperation operation) { operation.Applied = true; _shardLastOperationCache.AddOrUpdate(shardId, operation, (key, val) => operation); }
public async Task <bool> ReplicateShardWriteOperationAsync(ShardWriteOperation operation) { // Get the last operation ShardWriteOperation lastOperation; var startTime = DateTime.Now; if (operation.Pos != 1) { while ((lastOperation = await GetOrPopulateOperationCache(operation.Data.ShardId.Value)) == null || !lastOperation.Applied || lastOperation.Pos < operation.Pos - 1) { //Assume in the next 3 seconds you should receive the previos requests if ((DateTime.Now - startTime).TotalMilliseconds > 3000) { return(false); } await Task.Delay(100); } if (lastOperation.Pos != operation.Pos - 1) { lastOperation = await _shardRepository.GetShardWriteOperationAsync(operation.Data.ShardId.Value, operation.Pos - 1); } string newHash; if ((newHash = ObjectUtility.HashStrings(lastOperation.ShardHash, operation.Id)) != operation.ShardHash) { //Critical error with data concurrency, revert back to last known good commit via the resync process _logger.LogError(_nodeStateService.GetNodeLogId() + "Failed to check the hash for operation " + operation.Id + ", marking shard as out of sync..."); throw new Exception(_nodeStateService.GetNodeLogId() + "Failed to check the hash for operation " + operation.Id + ", marking shard as out of sync..."); } } // if (operation.Pos > 1) //{ // lock (_shardLastOperationCache[operation.Data.ShardId.Value]) // { //The operation must already be being applied /* if (_shardLastOperationCache[operation.Data.ShardId.Value].Pos >= operation.Pos) * { * throw new Exception("There must be a replication process being run"); * }*/ _shardRepository.AddShardWriteOperationAsync(operation).GetAwaiter().GetResult(); //Run shard operation ApplyOperationToDatastore(operation); _shardRepository.MarkShardWriteOperationAppliedAsync(operation.Id).GetAwaiter().GetResult(); //Update the cache UpdateOperationCache(operation.Data.ShardId.Value, operation); //} /* } * else * { * await _shardRepository.AddShardWriteOperationAsync(operation); * //Run shard operation * ApplyOperationToDatastore(operation); * await _shardRepository.MarkShardWriteOperationAppliedAsync(operation.Id); * //Update the cache * UpdateOperationCache(operation.Data.ShardId.Value, operation); * }*/ return(true); }
public async Task <bool> EnqueueOperationAsync(ShardWriteOperation data) { await _shardWriteOperationsQueue.InsertOneAsync(data); return(true); }
public async Task <bool> DeleteOperationFromQueueAsync(ShardWriteOperation operation) { return((await _shardWriteOperationsQueue.DeleteOneAsync(swoq => swoq.Id == operation.Id)).IsAcknowledged); }
public async Task <bool> AddOperationToTransitAsync(ShardWriteOperation operation) { await _shardWriteOperationsTransit.InsertOneAsync(operation); return(true); }
public Task <bool> AddShardWriteOperationAsync(ShardWriteOperation operation) { return(Task.FromResult(ShardWriteOperations.TryAdd(operation.Id, SystemExtension.Clone(operation)))); }
public Task <bool> AddOperationToTransitAsync(ShardWriteOperation operation) { return(Task.FromResult(TransitQueue.TryAdd(operation.Id, SystemExtension.Clone(operation)))); }
public async Task <bool> SyncShard(Guid shardId, string type) { var lastOperation = await _shardRepository.GetShardWriteOperationAsync(shardId, _shardRepository.GetTotalShardWriteOperationsCount(shardId)); int lastOperationPos = lastOperation == null ? 0 : lastOperation.Pos; var shardMetadata = _stateMachine.GetShard(type, shardId); var result = await _clusterClient.Send(shardMetadata.PrimaryAllocation, new RequestShardWriteOperations() { ShardId = shardId, From = lastOperationPos, To = lastOperationPos + 1 }); if (result.IsSuccessful) { var totalPositions = _shardRepository.GetTotalShardWriteOperationsCount(shardId); //If the primary has less operations if (result.LatestPosition < lastOperationPos) { while (totalPositions != result.LatestPosition) { _writer.ReverseLocalTransaction(shardId, type, totalPositions); totalPositions--; } } else { //Check whether the hash is equal, if not equal roll back each transaction var currentPosition = lastOperationPos; ShardWriteOperation currentOperation = null; if (lastOperationPos != 0 && !(result.Operations[lastOperationPos].ShardHash == lastOperation.ShardHash)) { //While the shard position does not match while (!((await _clusterClient.Send(shardMetadata.PrimaryAllocation, new RequestShardWriteOperations() { ShardId = shardId, From = currentPosition, To = currentPosition })).Operations[currentPosition].ShardHash != (currentOperation = await _shardRepository.GetShardWriteOperationAsync(shardId, currentPosition)).ShardHash)) { _logger.LogInformation("Reverting transaction " + currentOperation.Pos + " on shard " + shardId); _writer.ReverseLocalTransaction(shardId, type, currentOperation.Pos); currentPosition--; } } result = await _clusterClient.Send(shardMetadata.PrimaryAllocation, new RequestShardWriteOperations() { ShardId = shardId, From = currentPosition + 1, To = currentPosition + 50 }); var totalShards = (_shardRepository.GetTotalShardWriteOperationsCount(shardId)); //If you have more operations if (result.LatestPosition < totalShards) { _logger.LogWarning("Detected more nodes locally then primary, revering to latest position"); while (totalShards != result.LatestPosition) { _writer.ReverseLocalTransaction(shardId, type, totalShards); totalShards--; } } while (result.LatestPosition != (_shardRepository.GetTotalShardWriteOperationsCount(shardId))) { foreach (var operation in result.Operations) { _logger.LogDebug(_nodeStateService.Id + "Replicated operation " + operation.Key + " for shard " + shardId); await _writer.ReplicateShardWriteOperationAsync(operation.Value); } currentPosition = result.Operations.Last().Key; result = await _clusterClient.Send(shardMetadata.PrimaryAllocation, new RequestShardWriteOperations() { ShardId = shardId, From = currentPosition + 1, To = currentPosition + 50 }); } } _logger.LogInformation("Successfully recovered data on shard " + shardId); return(true); } else { throw new Exception("Failed to fetch shard from primary."); } }