public WriteObjectReply Write(WriteObjectRequest request)
{
Console.WriteLine("Received write with params:");
Console.WriteLine($"Partition_id: {request.Key.PartitionId}");
Console.WriteLine($"Object_id: {request.Key.ObjectId}");
Console.WriteLine($"Value: {request.Value}");
if (MasteredPartitions.Contains(request.Key.PartitionId))
{
lock (WriteGlobalLock)
{
// I'm master of this object's partition
// Send request to all other servers of partition
ServersByPartition.TryGetValue(request.Key.PartitionId, out List <string> serverIds);
if (!KeyValuePairs.TryGetValue(new ObjectKey(request.Key), out ObjectValueManager objectValueManager))
{
LocalReadWriteLock.AcquireWriterLock(-1);
objectValueManager = new ObjectValueManager();
KeyValuePairs[new ObjectKey(request.Key)] = objectValueManager;
objectValueManager.LockWrite();
LocalReadWriteLock.ReleaseWriterLock();
}
else
{
objectValueManager.LockWrite();
}
var connectionCrashedServers = new HashSet <string>();
foreach (var server in ServerUrls.Where(x => serverIds.Contains(x.Key) && x.Key != MyId))
{
var channel = GrpcChannel.ForAddress(server.Value);
var client = new ServerSyncGrpcService.ServerSyncGrpcServiceClient(channel);
// What to do if success returns false ?
try
{
client.LockObject(new LockObjectRequest
{
Key = request.Key
});
} catch (RpcException e)
{
// If grpc does no respond, we can assume it has crashed
if (e.Status.StatusCode == StatusCode.DeadlineExceeded || e.Status.StatusCode == StatusCode.Unavailable || e.Status.StatusCode == StatusCode.Internal)
{
// Add to hash Set
Console.WriteLine($"Server {server.Key} has crashed");
connectionCrashedServers.Add(server.Key);
}
else
{
throw e;
}
}
}
foreach (var server in ServerUrls.Where(x => serverIds.Contains(x.Key) && x.Key != MyId))
{
try
{
var channel = GrpcChannel.ForAddress(server.Value);
var client = new ServerSyncGrpcService.ServerSyncGrpcServiceClient(channel);
// What to do if success returns false ?
client.ReleaseObjectLock(new ReleaseObjectLockRequest
{
Key = request.Key,
Value = request.Value
});
}
catch (RpcException e)
{
if (e.Status.StatusCode == StatusCode.DeadlineExceeded || e.Status.StatusCode == StatusCode.Unavailable || e.Status.StatusCode == StatusCode.Internal)
{
// Add to hash Set
Console.WriteLine($"Server {server.Key} has crashed");
connectionCrashedServers.Add(server.Key);
}
else
{
throw e;
}
}
}
if (connectionCrashedServers.Any())
{
// Update the crashed servers
UpdateCrashedServers(request.Key.PartitionId, connectionCrashedServers);
// Contact Partition slaves an update their view of the partition
foreach (var server in ServerUrls.Where(x => serverIds.Contains(x.Key) && x.Key != MyId))
{
var channel = GrpcChannel.ForAddress(server.Value);
var client = new ServerSyncGrpcService.ServerSyncGrpcServiceClient(channel);
client.RemoveCrashedServers(new RemoveCrashedServersRequest
{
PartitionId = request.Key.PartitionId,
ServerIds = { connectionCrashedServers }
});
}
}
objectValueManager.UnlockWrite(request.Value);
return(new WriteObjectReply
{
Ok = true
});
}
}
else
{
// Tell him I'm not the master
throw new RpcException(new Status(StatusCode.PermissionDenied, $"Server {MyId} is not the master of partition {request.Key.PartitionId}"));
}
}
public void WriteObject(string partition_id, string object_id, string value)
{
int currentServerPartitionIndex;
List <string> ServersOfPartition = ServersIdByPartition[partition_id];
if (ServersOfPartition.Count == 0)
{
Console.WriteLine($"No available servers for partition {partition_id}");
return;
}
// Check if connected to server with desired partition
if (!ServersOfPartition.Contains(currentServerId))
{
// If not connect to first server of partition
TryChangeCommunicationChannel(ServersOfPartition[0]);
currentServerPartitionIndex = 0;
}
else
{
currentServerPartitionIndex = ServersOfPartition.IndexOf(currentServerId);
}
var success = false;
int numTries = 0;
WriteObjectRequest request = new WriteObjectRequest
{
Key = new Key
{
PartitionId = partition_id,
ObjectId = object_id
},
Value = value
};
var crashedServers = new ConcurrentBag <string>();
while (!success && numTries < ServersOfPartition.Count)
{
try
{
var reply = Client.WriteObject(request);
Console.WriteLine("Received: " + reply.Ok);
success = true;
} catch (RpcException e)
{
if (e.Status.StatusCode == StatusCode.PermissionDenied)
{
Console.WriteLine($"Cannot write in server {currentServerId}");
}
else
{
// If error is because Server failed, keep it
if (e.Status.StatusCode == StatusCode.Unavailable || e.Status.StatusCode == StatusCode.DeadlineExceeded || e.Status.StatusCode == StatusCode.Internal)
{
Console.WriteLine($"Server {currentServerId} is down");
crashedServers.Add(currentServerId);
}
else
{
throw e;
}
}
if (++numTries < ServersOfPartition.Count)
{
// Connect to next server in list
currentServerPartitionIndex = (currentServerPartitionIndex + 1) % ServersOfPartition.Count;
TryChangeCommunicationChannel(ServersOfPartition[currentServerPartitionIndex]);
}
}
}
// Remove crashed servers from list and update CrashedServers list
CrashedServers.Union(crashedServers);
foreach (var crashedServer in crashedServers)
{
foreach (var kvPair in ServersIdByPartition)
{
if (kvPair.Value.Contains(crashedServer))
{
kvPair.Value.Remove(crashedServer);
}
}
}
}
// Write Object
public override Task <WriteObjectReply> WriteObject(WriteObjectRequest request, ServerCallContext context)
{
return(Task.FromResult(Write(request)));
}