public WriteObjectReply Write(WriteObjectRequest request) { Console.WriteLine("Received write with params:"); Console.WriteLine($"Partition_id: {request.Key.PartitionId}"); Console.WriteLine($"Object_id: {request.Key.ObjectId}"); Console.WriteLine($"Value: {request.Value}"); if (MasteredPartitions.Contains(request.Key.PartitionId)) { lock (WriteGlobalLock) { // I'm master of this object's partition // Send request to all other servers of partition ServersByPartition.TryGetValue(request.Key.PartitionId, out List <string> serverIds); if (!KeyValuePairs.TryGetValue(new ObjectKey(request.Key), out ObjectValueManager objectValueManager)) { LocalReadWriteLock.AcquireWriterLock(-1); objectValueManager = new ObjectValueManager(); KeyValuePairs[new ObjectKey(request.Key)] = objectValueManager; objectValueManager.LockWrite(); LocalReadWriteLock.ReleaseWriterLock(); } else { objectValueManager.LockWrite(); } var connectionCrashedServers = new HashSet <string>(); foreach (var server in ServerUrls.Where(x => serverIds.Contains(x.Key) && x.Key != MyId)) { var channel = GrpcChannel.ForAddress(server.Value); var client = new ServerSyncGrpcService.ServerSyncGrpcServiceClient(channel); // What to do if success returns false ? try { client.LockObject(new LockObjectRequest { Key = request.Key }); } catch (RpcException e) { // If grpc does no respond, we can assume it has crashed if (e.Status.StatusCode == StatusCode.DeadlineExceeded || e.Status.StatusCode == StatusCode.Unavailable || e.Status.StatusCode == StatusCode.Internal) { // Add to hash Set Console.WriteLine($"Server {server.Key} has crashed"); connectionCrashedServers.Add(server.Key); } else { throw e; } } } foreach (var server in ServerUrls.Where(x => serverIds.Contains(x.Key) && x.Key != MyId)) { try { var channel = GrpcChannel.ForAddress(server.Value); var client = new ServerSyncGrpcService.ServerSyncGrpcServiceClient(channel); // What to do if success returns false ? client.ReleaseObjectLock(new ReleaseObjectLockRequest { Key = request.Key, Value = request.Value }); } catch (RpcException e) { if (e.Status.StatusCode == StatusCode.DeadlineExceeded || e.Status.StatusCode == StatusCode.Unavailable || e.Status.StatusCode == StatusCode.Internal) { // Add to hash Set Console.WriteLine($"Server {server.Key} has crashed"); connectionCrashedServers.Add(server.Key); } else { throw e; } } } if (connectionCrashedServers.Any()) { // Update the crashed servers UpdateCrashedServers(request.Key.PartitionId, connectionCrashedServers); // Contact Partition slaves an update their view of the partition foreach (var server in ServerUrls.Where(x => serverIds.Contains(x.Key) && x.Key != MyId)) { var channel = GrpcChannel.ForAddress(server.Value); var client = new ServerSyncGrpcService.ServerSyncGrpcServiceClient(channel); client.RemoveCrashedServers(new RemoveCrashedServersRequest { PartitionId = request.Key.PartitionId, ServerIds = { connectionCrashedServers } }); } } objectValueManager.UnlockWrite(request.Value); return(new WriteObjectReply { Ok = true }); } } else { // Tell him I'm not the master throw new RpcException(new Status(StatusCode.PermissionDenied, $"Server {MyId} is not the master of partition {request.Key.PartitionId}")); } }
public void WriteObject(string partition_id, string object_id, string value) { int currentServerPartitionIndex; List <string> ServersOfPartition = ServersIdByPartition[partition_id]; if (ServersOfPartition.Count == 0) { Console.WriteLine($"No available servers for partition {partition_id}"); return; } // Check if connected to server with desired partition if (!ServersOfPartition.Contains(currentServerId)) { // If not connect to first server of partition TryChangeCommunicationChannel(ServersOfPartition[0]); currentServerPartitionIndex = 0; } else { currentServerPartitionIndex = ServersOfPartition.IndexOf(currentServerId); } var success = false; int numTries = 0; WriteObjectRequest request = new WriteObjectRequest { Key = new Key { PartitionId = partition_id, ObjectId = object_id }, Value = value }; var crashedServers = new ConcurrentBag <string>(); while (!success && numTries < ServersOfPartition.Count) { try { var reply = Client.WriteObject(request); Console.WriteLine("Received: " + reply.Ok); success = true; } catch (RpcException e) { if (e.Status.StatusCode == StatusCode.PermissionDenied) { Console.WriteLine($"Cannot write in server {currentServerId}"); } else { // If error is because Server failed, keep it if (e.Status.StatusCode == StatusCode.Unavailable || e.Status.StatusCode == StatusCode.DeadlineExceeded || e.Status.StatusCode == StatusCode.Internal) { Console.WriteLine($"Server {currentServerId} is down"); crashedServers.Add(currentServerId); } else { throw e; } } if (++numTries < ServersOfPartition.Count) { // Connect to next server in list currentServerPartitionIndex = (currentServerPartitionIndex + 1) % ServersOfPartition.Count; TryChangeCommunicationChannel(ServersOfPartition[currentServerPartitionIndex]); } } } // Remove crashed servers from list and update CrashedServers list CrashedServers.Union(crashedServers); foreach (var crashedServer in crashedServers) { foreach (var kvPair in ServersIdByPartition) { if (kvPair.Value.Contains(crashedServer)) { kvPair.Value.Remove(crashedServer); } } } }
// Write Object public override Task <WriteObjectReply> WriteObject(WriteObjectRequest request, ServerCallContext context) { return(Task.FromResult(Write(request))); }