public async Task <AddShardWriteOperationResponse> AddShardWriteOperationHandler(AddShardWriteOperation request) { var startDate = DateTime.Now; var checkpoint = 1; var totalOperation = _writeCache.OperationQueue.Count(); _logger.LogDebug(_nodeStateService.GetNodeLogId() + "Received write request for object " + request.Data.Id + " for request " + request.Data.ShardId); AddShardWriteOperationResponse finalResult = new AddShardWriteOperationResponse(); //Check if index exists, if not - create one if (!_stateMachine.IndexExists(request.Data.ShardType)) { if (IndexCreationQueue.Where(icq => icq == request.Data.ShardType).Count() > 0) { while (!_stateMachine.IndexExists(request.Data.ShardType)) { if ((DateTime.Now - startDate).Milliseconds > _clusterOptions.DataTransferTimeoutMs) { throw new IndexCreationFailedException("Index creation for shard " + request.Data.ShardType + " is already queued."); } await Task.Delay(10); } } else { await _clusterClient.Send(_nodeStateService.CurrentLeader.Value, new RequestCreateIndex() { Type = request.Data.ShardType }); DateTime startIndexCreation = DateTime.Now; while (!_stateMachine.IndexExists(request.Data.ShardType)) { if ((DateTime.Now - startIndexCreation).Milliseconds > _clusterOptions.DataTransferTimeoutMs) { throw new IndexCreationFailedException("Index creation for shard " + request.Data.ShardType + " timed out."); } await Task.Delay(100); } } } if (_nodeOptions.EnablePerformanceLogging) { PerformanceMetricUtility.PrintCheckPoint(ref totalLock, ref totals, ref startDate, ref checkpoint, "index allocation"); } ShardAllocationMetadata shardMetadata; if (request.Data.ShardId == null) { var allocations = _stateMachine.GetShards(request.Data.ShardType); Random rand = new Random(); var selectedNodeIndex = rand.Next(0, allocations.Length); request.Data.ShardId = allocations[selectedNodeIndex].Id; shardMetadata = allocations[selectedNodeIndex]; } else { shardMetadata = _stateMachine.GetShard(request.Data.ShardType, request.Data.ShardId.Value); } if (_nodeOptions.EnablePerformanceLogging) { PerformanceMetricUtility.PrintCheckPoint(ref totalLock, ref totals, ref startDate, ref checkpoint, "shard Allocation"); } //If the shard is assigned to you if (shardMetadata.PrimaryAllocation == _nodeStateService.Id) { var operationId = Guid.NewGuid().ToString(); finalResult.OperationId = operationId; await _writeCache.EnqueueOperationAsync(new ShardWriteOperation() { Data = request.Data, Id = operationId, Operation = request.Operation, TransactionDate = DateTime.Now }); finalResult.IsSuccessful = true; if (_nodeOptions.EnablePerformanceLogging) { PerformanceMetricUtility.PrintCheckPoint(ref totalLock, ref totals, ref startDate, ref checkpoint, "enqueue"); } ShardWriteOperation transaction; while (!_writeCache.IsOperationComplete(operationId)) { if ((DateTime.Now - startDate).Milliseconds > _clusterOptions.DataTransferTimeoutMs) { throw new IndexCreationFailedException("Queue clearance for transaction " + operationId + request.Data.ShardType + " timed out."); } await Task.Delay(totalOperation); } // printCheckPoint(ref startDate, ref checkpoint, "wait for completion"); } else { try { return(await _clusterClient.Send(shardMetadata.PrimaryAllocation, request)); } catch (Exception e) { _logger.LogError(_nodeStateService.GetNodeLogId() + "Failed to write " + request.Operation.ToString() + " request to primary node " + _stateMachine.CurrentState.Nodes[shardMetadata.PrimaryAllocation].TransportAddress + " for object " + request.Data.Id + " shard " + request.Data.ShardId + "|" + e.StackTrace); throw e; } } if (request.RemoveLock) { var result = await _clusterClient.Send(new ExecuteCommands() { Commands = new List <BaseCommand>() { new RemoveLock() { Name = request.Data.GetLockName(), LockId = request.LockId } }, WaitForCommits = true }); if (result.IsSuccessful) { finalResult.LockRemoved = true; } } Interlocked.Increment(ref totalRequests); if (_nodeOptions.EnablePerformanceLogging) { PerformanceMetricUtility.PrintCheckPoint(ref totalLock, ref totals, ref startDate, ref checkpoint, "removeLock"); } return(finalResult); }