public async Task Write(GStoreObjectIdentifier gStoreObjectIdentifier, string newValue)
{
try
{
if (!connectionManager.IsMasterForPartition(gStoreObjectIdentifier.PartitionId))
{
throw new Exception("Not master");
}
// Acquire lock on local object
ReaderWriterLockEnhancedSlim objectLock = GetObjectLock(gStoreObjectIdentifier);
int lockId = objectLock.EnterWriteLock();
// Send lock requests to all remote objects
IDictionary <string, int> replicaLocks = await LockController.ExecuteAsync(connectionManager, gStoreObjectIdentifier);
// Once lock confirmations arrive, write to local object and unlock it
GStoreObject gStoreObject = AddOrUpdate(gStoreObjectIdentifier, newValue);
objectLock.ExitWriteLock(lockId);
// Send write requests to all remote objects
await WriteReplicaController.ExecuteAsync(connectionManager, gStoreObject, replicaLocks);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public static async Task ExecuteAsync(ConnectionManager connectionManager, GStoreObject gStoreObject, int version)
{
GStoreObjectIdentifier gStoreObjectIdentifier = gStoreObject.Identifier;
// Get all replicas associated to this Partition
IImmutableSet <Server> servers = connectionManager.GetAliveServers(gStoreObjectIdentifier.PartitionId);
IDictionary <string, Task> writeTasks = new Dictionary <string, Task>();
foreach (Server server in servers)
{
if (server.Id != connectionManager.SelfServerId)
{
writeTasks.Add(server.Id, ExecuteServerAsync(connectionManager, server.Stub, gStoreObject, version));
}
}
foreach (KeyValuePair <string, Task> writeTaskPair in writeTasks)
{
string serverId = writeTaskPair.Key;
try
{
await writeTaskPair.Value;
}
catch (Grpc.Core.RpcException e) when(e.StatusCode == Grpc.Core.StatusCode.Internal)
{
connectionManager.DeclareDead(serverId);
}
}
}
private static GStoreObjectReplica CreateObjectReplica(DataObjectReplica dataObjectReplica)
{
GStoreObjectIdentifier gStoreObjectIdentifier = new GStoreObjectIdentifier(dataObjectReplica.Object.ObjectIdentifier.PartitionId, dataObjectReplica.Object.ObjectIdentifier.ObjectId);
GStoreObject gStoreObject = new GStoreObject(gStoreObjectIdentifier, dataObjectReplica.Object.Value);
return(new GStoreObjectReplica(gStoreObject, dataObjectReplica.IsMasterReplica));
}
private Empty ExecuteWrite(WriteRequest request)
{
Console.WriteLine($"Write Replica request -> PartitionId: {request.Object.ObjectIdentifier.PartitionId} ObjectId: {request.Object.ObjectIdentifier.ObjectId} Value: {request.Object.Value} LockId: {request.LockId}");
GStoreObjectIdentifier gStoreObjectIdentifier = new GStoreObjectIdentifier(request.Object.ObjectIdentifier.PartitionId, request.Object.ObjectIdentifier.ObjectId);
gStore.WriteReplica(gStoreObjectIdentifier, request.Object.Value, request.LockId);
return(new Empty());
}
private GStoreObjectVersioning GetObjectVersioning(GStoreObjectIdentifier gStoreObjectIdentifier)
{
return(ObjectVersionings.GetOrAdd(gStoreObjectIdentifier,
(key) =>
{
return new GStoreObjectVersioning();
}));
}
private ReaderWriterLockEnhancedSlim GetObjectLock(GStoreObjectIdentifier gStoreObjectIdentifier)
{
return(ObjectLocks.GetOrAdd(gStoreObjectIdentifier,
(key) =>
{
return new ReaderWriterLockEnhancedSlim();
}));
}
private async Task <Empty> ExecuteWrite(GStoreWriteRequest request)
{
Console.WriteLine($"Write request -> PartitionId: {request.Object.ObjectIdentifier.PartitionId} ObjectId: {request.Object.ObjectIdentifier} Value: {request.Object.Value}");
GStoreObjectIdentifier gStoreObjectIdentifier = new GStoreObjectIdentifier(request.Object.ObjectIdentifier.PartitionId, request.Object.ObjectIdentifier.ObjectId);
await gStore.Write(gStoreObjectIdentifier, request.Object.Value);
return(new Empty());
}
public int Lock(GStoreObjectIdentifier gStoreObjectIdentifier)
{
ReaderWriterLockEnhancedSlim objectLock = GetObjectLock(gStoreObjectIdentifier);
string masterId = connectionManager.GetPartitionMasterId(gStoreObjectIdentifier.PartitionId);
int lockId = objectLock.EnterWriteLock();
ConcurrentDictionary <int, GStoreObjectIdentifier> locks = MastersLocks.GetOrAdd(masterId, (masterId) => new ConcurrentDictionary <int, GStoreObjectIdentifier>());
locks.TryAdd(lockId, gStoreObjectIdentifier);
return(lockId);
}
public int GetObjectVersionNumber(GStoreObjectIdentifier gStoreObjectIdentifier)
{
int version = ObjectVersionNumber.GetOrAdd(gStoreObjectIdentifier,
(key) =>
{
return(0);
});
return(version);
}
private LockReply ExecuteLock(LockRequest request)
{
Console.WriteLine($"Lock request -> PartitionId: {request.ObjectIdentifier.PartitionId} ObjectId: {request.ObjectIdentifier.ObjectId}");
GStoreObjectIdentifier gStoreObjectIdentifier = new GStoreObjectIdentifier(request.ObjectIdentifier.PartitionId, request.ObjectIdentifier.ObjectId);
int lockId = gStore.Lock(gStoreObjectIdentifier);
return(new LockReply
{
LockId = lockId
});
}
private void AddOrUpdateObjectVersionServerWriter(GStoreObjectIdentifier gStoreObjectIdentifier, string serverId)
{
_ = ObjectVersionServerWriter.AddOrUpdate(gStoreObjectIdentifier,
(key) =>
{
return(serverId);
},
(key, value) =>
{
return(serverId);
});
}
private void SetObjectVersionNumber(GStoreObjectIdentifier gStoreObjectIdentifier, int version)
{
_ = ObjectVersionNumber.AddOrUpdate(gStoreObjectIdentifier,
(key) =>
{
return(version);
},
(key, value) =>
{
return(version);
}
);
}
private GStoreObject AddOrUpdate(GStoreObjectIdentifier gStoreObjectIdentifier, string newValue)
{
return(DataStore.AddOrUpdate(gStoreObjectIdentifier,
(id) =>
{
return new GStoreObject(id, newValue);
},
(id, gStoreObject) =>
{
gStoreObject.Value = newValue;
return gStoreObject;
}));
}
public string Read(GStoreObjectIdentifier gStoreObjectIdentifier)
{
ReaderWriterLockEnhancedSlim objectLock = GetObjectLock(gStoreObjectIdentifier);
string value = null;
int id = objectLock.EnterReadLock();
if (DataStore.TryGetValue(gStoreObjectIdentifier, out GStoreObject gStoreObject))
{
value = gStoreObject.Value;
}
objectLock.ExitReadLock(id);
return(value);
}
private GStoreReadReply ExecuteRead(GStoreReadRequest request)
{
Console.WriteLine($"Read request -> PartitionId: {request.ObjectIdentifier.PartitionId} ObjectId: {request.ObjectIdentifier.ObjectId}");
GStoreObjectIdentifier gStoreObjectIdentifier = new GStoreObjectIdentifier(request.ObjectIdentifier.PartitionId, request.ObjectIdentifier.ObjectId);
string value = gStore.Read(gStoreObjectIdentifier);
if (value == null)
{
value = "N/A";
}
return(new GStoreReadReply
{
Object = DataObjectBuilder.FromObjectIdentifier(request.ObjectIdentifier, value)
});
}
private int GetAndIncrementObjectVersionNumber(GStoreObjectIdentifier gStoreObjectIdentifier)
{
int version = ObjectVersionNumber.AddOrUpdate(gStoreObjectIdentifier,
(key) =>
{
return(1);
},
(key, value) =>
{
return(++value);
}
);
return(version);
}
public void WriteReplica(GStoreObjectIdentifier gStoreObjectIdentifier, string newValue, int lockId)
{
ReaderWriterLockEnhancedSlim objectLock = GetObjectLock(gStoreObjectIdentifier);
if (!objectLock.IsWriteLockValid(lockId))
{
// throw error
}
AddOrUpdate(gStoreObjectIdentifier, newValue);
objectLock.ExitWriteLock(lockId);
string masterId = connectionManager.GetPartitionMasterId(gStoreObjectIdentifier.PartitionId);
MastersLocks.TryGetValue(masterId, out ConcurrentDictionary <int, GStoreObjectIdentifier> locks);
locks.TryRemove(lockId, out _);
}
public async Task CleanLocks(string masterId)
{
MastersLocks.TryGetValue(masterId, out ConcurrentDictionary <int, GStoreObjectIdentifier> masterLocks);
if (masterLocks == null)
{
return;
}
IDictionary <int, Task <string> > tasks = new Dictionary <int, Task <string> >();
foreach (KeyValuePair <int, GStoreObjectIdentifier> locks in masterLocks)
{
int lockId = locks.Key;
GStoreObjectIdentifier gStoreObjectIdentifier = locks.Value;
ReaderWriterLockEnhancedSlim objectLock = GetObjectLock(gStoreObjectIdentifier);
if (!objectLock.IsWriteLockValid(lockId))
{
throw new Exception("Invalid lock");
}
Task <string> task = ReadRecoveryController.ExecuteAsync(connectionManager, gStoreObjectIdentifier);
tasks.Add(lockId, task);
}
foreach (KeyValuePair <int, Task <string> > taskPair in tasks)
{
Task <string> task = taskPair.Value;
int lockId = taskPair.Key;
masterLocks.TryGetValue(lockId, out GStoreObjectIdentifier gStoreObjectIdentifier);
ReaderWriterLockEnhancedSlim objectLock = GetObjectLock(gStoreObjectIdentifier);
string value = await task;
if (value != null)
{
_ = AddOrUpdate(gStoreObjectIdentifier, value);
}
Console.WriteLine("Cleaned one lock");
objectLock.ExitWriteLock(lockId);
}
}
public string Read(GStoreObjectIdentifier gStoreObjectIdentifier)
{
ReaderWriterLockSlim objectLock = GetObjectLock(gStoreObjectIdentifier);
objectLock.EnterReadLock();
try
{
string value = null;
if (DataStore.TryGetValue(gStoreObjectIdentifier, out GStoreObject gStoreObject))
{
value = gStoreObject.Value;
}
return(value);
}
finally
{
objectLock.ExitReadLock();
}
}
public void WriteReplica(GStoreObjectIdentifier gStoreObjectIdentifier, string newValue, string newServerId, int newVersion)
{
ReaderWriterLockSlim objectLock = GetObjectLock(gStoreObjectIdentifier);
objectLock.EnterWriteLock();
try
{
int version = GetObjectVersionNumber(gStoreObjectIdentifier);
string serverId = GetObjectVersionServerWriter(gStoreObjectIdentifier);
if (newVersion > version || newVersion == version && string.Compare(newServerId, serverId) < 0)
{
_ = AddOrUpdate(gStoreObjectIdentifier, newValue);
SetObjectVersionNumber(gStoreObjectIdentifier, newVersion);
}
}
finally
{
objectLock.ExitWriteLock();
}
}
public ICollection <GStoreObject> ReadAll()
{
ISet <GStoreObject> values = new HashSet <GStoreObject>();
foreach (KeyValuePair <GStoreObjectIdentifier, ReaderWriterLockSlim> objectPair in ObjectLocks)
{
GStoreObjectIdentifier gStoreObjectIdentifier = objectPair.Key;
ReaderWriterLockSlim objectLock = objectPair.Value;
objectLock.EnterReadLock();
try
{
DataStore.TryGetValue(gStoreObjectIdentifier, out GStoreObject gStoreObject);
values.Add(gStoreObject);
}
finally
{
objectLock.ExitReadLock();
}
}
return(values);
}
public void Write(GStoreObjectIdentifier gStoreObjectIdentifier, string newValue)
{
// Acquire lock on local object
ReaderWriterLockSlim objectLock = GetObjectLock(gStoreObjectIdentifier);
objectLock.EnterWriteLock();
try
{
int version = GetAndIncrementObjectVersionNumber(gStoreObjectIdentifier);
AddOrUpdateObjectVersionServerWriter(gStoreObjectIdentifier, connectionManager.SelfServerId);
GStoreObject gStoreObject = AddOrUpdate(gStoreObjectIdentifier, newValue);
_ = WriteReplicaController.ExecuteAsync(connectionManager, gStoreObject, version);
}
catch (Exception e)
{
Console.WriteLine(e);
}
finally
{
objectLock.ExitWriteLock();
}
}
private ReadRecoveryReply ExecuteReadRecovery(ReadRecoveryRequest request)
{
GStoreObjectIdentifier gStoreObjectIdentifier = new GStoreObjectIdentifier(request.ObjectIdentifier.PartitionId, request.ObjectIdentifier.ObjectId);
bool valid = !gStore.IsLocked(gStoreObjectIdentifier);
ObjectDto objectDto = null;
if (valid)
{
objectDto = new ObjectDto
{
ObjectIdentifier = request.ObjectIdentifier,
Value = gStore.Read(gStoreObjectIdentifier)
};
}
return(new ReadRecoveryReply
{
Valid = valid,
Object = objectDto
});
}
public static async Task <string> ExecuteAsync(ConnectionManager connectionManager, GStoreObjectIdentifier gStoreObjectIdentifier)
{
string partitionId = gStoreObjectIdentifier.PartitionId;
IImmutableSet <Server> partitionServers = connectionManager.GetAliveServers(partitionId);
List <Task <ReadRecoveryReply> > tasks = new List <Task <ReadRecoveryReply> >();
foreach (Server server in partitionServers)
{
Task <ReadRecoveryReply> t = ExecuteServerAsync(server.Stub, gStoreObjectIdentifier);
tasks.Add(t);
}
foreach (Task <ReadRecoveryReply> t in tasks)
{
ReadRecoveryReply reply = await t;
if (reply.Valid)
{
return(reply.Object.Value);
}
}
return(null);
}
private static async Task <ReadRecoveryReply> ExecuteServerAsync(MasterReplicaService.MasterReplicaServiceClient Stub, GStoreObjectIdentifier gStoreObjectIdentifier)
{
ReadRecoveryRequest request = new ReadRecoveryRequest
{
ObjectIdentifier = new ObjectIdentifierDto
{
PartitionId = gStoreObjectIdentifier.PartitionId,
ObjectId = gStoreObjectIdentifier.ObjectId
}
};
return(await Stub.ReadRecoveryAsync(request));
}
public static async Task <IDictionary <string, int> > ExecuteAsync(ConnectionManager connectionManager, GStoreObjectIdentifier gStoreObjectIdentifier)
{
// Get all replicas associated to this Partition
IImmutableSet <Server> replicas = connectionManager.GetPartitionAliveReplicas(gStoreObjectIdentifier.PartitionId);
IDictionary <string, Task <int> > lockTasks = new Dictionary <string, Task <int> >();
foreach (Server replica in replicas)
{
lockTasks.Add(replica.Id, ExecuteReplicaAsync(replica.Stub, gStoreObjectIdentifier));
}
// Await for lock requests and save their values
IDictionary <string, int> lockValues = new Dictionary <string, int>();
foreach (KeyValuePair <string, Task <int> > lockTaskPair in lockTasks)
{
string replicaId = lockTaskPair.Key;
try
{
lockValues.Add(replicaId, await lockTaskPair.Value);
}
catch (Grpc.Core.RpcException e) when(e.StatusCode == Grpc.Core.StatusCode.Internal)
{
connectionManager.DeclareDead(replicaId);
}
}
return(lockValues);
}
private static async Task <int> ExecuteReplicaAsync(MasterReplicaService.MasterReplicaServiceClient Stub, GStoreObjectIdentifier gStoreObjectIdentifier)
{
LockRequest lockRequest = new LockRequest
{
ObjectIdentifier = new ObjectIdentifierDto
{
PartitionId = gStoreObjectIdentifier.PartitionId,
ObjectId = gStoreObjectIdentifier.ObjectId
}
};
LockReply lockReply = await Stub.LockAsync(lockRequest);
return(lockReply.LockId);
}
internal static DataObjectIdentifier FromObjectIdentifier(GStoreObjectIdentifier objectIdentifier)
{
return(FromString(objectIdentifier.PartitionId, objectIdentifier.ObjectId));
}
private static GStoreObject CreateObject(DataObject dataObject)
{
GStoreObjectIdentifier gStoreObjectIdentifier = new GStoreObjectIdentifier(dataObject.ObjectIdentifier.PartitionId, dataObject.ObjectIdentifier.ObjectId);
return(new GStoreObject(gStoreObjectIdentifier, dataObject.Value));
}
public bool IsLocked(GStoreObjectIdentifier gStoreObjectIdentifier)
{
ReaderWriterLockEnhancedSlim objectLock = GetObjectLock(gStoreObjectIdentifier);
return(objectLock.IsWriteLocked());
}
