public NotifyCrashReply NotifyCrashHandler(NotifyCrashRequest request)
{
lock (_syncRoot)
{
string partitionName = request.PartitionId;
string crashedMasterServerId = request.CrashedMasterServerId;
string currentMasterServerId = PartitionMapping.GetPartitionMaster(partitionName);
Console.WriteLine(">>> Received Message from the client about crashed server: PartitionName=" + partitionName + ", CrashedServerId=" + crashedMasterServerId);
// check if the crashed server has been dealt with already
string[] serversOfThePartition = PartitionMapping.partitionMapping[partitionName];
if (!serversOfThePartition.Contains(crashedMasterServerId))
{
// this means an election already happened and the crashed server was deleted. the current partition master is the election result
// another possibility is that the master detected a replica failure. In this case the partition master didn't actually change saying that it did no harm.
Console.WriteLine(">>> Election Process Has Already Happened, CurrentMasterServerId=" + currentMasterServerId);
return(new NotifyCrashReply
{
Status = "OK",
MasterId = currentMasterServerId
});
}
// confirm that server is actually crashed
SendValueToReplica svr = new SendValueToReplica(server);
return(svr.HandleServerCrash(partitionName, crashedMasterServerId));
}
}
private bool NotifyPartitionsAboutCrashedServerAndReattachNewMaster(string crashedServerId)
{
List <string> partitions = PartitionMapping.GetPartitionsThatContainServer(crashedServerId);
foreach (string partitionName in partitions)
{
Console.WriteLine(">>> Notify Servers in Partition about Crash...");
Console.WriteLine(">>> Partition=" + partitionName);
string[] partitionNodes = PartitionMapping.GetPartitionAllNodes(partitionName);
bool newMasterWasReattachedSuccessfully = false;
foreach (string serverId in partitionNodes)
{
if (serverId == crashedServerId)
{
continue; // Skip crashed server
}
else if (TryNotifyServerAboutCrashedServer(partitionName, serverId, crashedServerId))
{
newMasterWasReattachedSuccessfully = true;
break;
}
}
return(newMasterWasReattachedSuccessfully);
}
return(false);
}
private bool TryNotifyServerAboutCrashedServer(string partitionName, string serverId, string crashedServerId)
{
Console.WriteLine(">>> Notifying Server=" + serverId + " about CrashedServer=" + crashedServerId);
try
{
reattachServer(serverId);
NotifyCrashReply notifyCrashReply = client.NotifyCrash(new NotifyCrashRequest {
PartitionId = partitionName, CrashedMasterServerId = crashedServerId
});
Console.WriteLine(">>> Got Reply from ServerId=" + serverId);
Console.WriteLine(">>> New Partition Master: PartitionName=" + partitionName + "PartitionMasterId=" + notifyCrashReply.MasterId);
string masterId = notifyCrashReply.MasterId;
PartitionMapping.SetPartitionMaster(partitionName, masterId);
if (serverId != masterId)
{
Console.WriteLine(">>> Reataching to new master. MasterId=" + masterId);
reattachServer(masterId);
}
else
{
Console.WriteLine(">>> Already Attached to new master. MasterId=" + masterId);
}
return(true);
}
catch
{
Console.WriteLine(">>> No Reply...");
return(false);
}
}
// Change name to reflect boolean return
private bool CompareClock(string partition_id, int reply_clock, DataStoreKeyDto object_key)
{
// Do Clock related staff
int partition_highest_clock = PartitionMapping.partitionToClockMapping[partition_id];
Console.WriteLine(">>> PartitionClock=" + partition_highest_clock + ", ReplyClock=" + reply_clock);
if (reply_clock > partition_highest_clock)
{
Console.WriteLine(">>> ReplyClock > PartitionClock. Reply clock is acceptable. ");
PartitionMapping.UpdatePartitionClock(partition_id, reply_clock);
return(true);
}
else if (reply_clock < partition_highest_clock)
{
Console.WriteLine(">>> ReplyClock < PartitionClock. Waiting " + retry_time.ToString() + "And trying again...");
wait(retry_time);
return(TryReadValue(object_key, partition_id));
}
else if (reply_clock == partition_highest_clock)
{
Console.WriteLine(">>> PartitionClock == ReplyClock. Reply clock is acceptable. ");
}
return(true);
}
public void createPartition(string partition_id)
{
String master_id = PartitionMapping.getPartitionMaster(partition_id);
bool is_master = master_id.Equals(server_id);
Partition p = new Partition(partition_id, is_master);
partitions.Add(p);
}
public void CreateLocalPartitions()
{
List <string> partitions = PartitionMapping.GetPartitionsByServerID(serverId);
foreach (string partition_id in partitions)
{
server.createPartition(partition_id);
}
}
public void createPartition(string partition_id)
{
string master_id = PartitionMapping.GetPartitionMaster(partition_id);
bool is_master = master_id.Equals(server_id);
int partition_clock = PartitionMapping.GetPartitionClock(partition_id);
Partition p = new Partition(partition_id, is_master, partition_clock);
partitions.Add(p);
}
private void write(string partition_id, string object_id, string value)
{
WriteReply reply;
if (debug_console)
{
Console.WriteLine("Get Partition Master from Partition Named: " + partition_id);
}
string partition_master_server_id = PartitionMapping.GetPartitionMaster(partition_id);
if (debug_console)
{
Console.WriteLine("Partition Master Server ID: " + partition_master_server_id);
}
reattachServer(partition_master_server_id);
var object_key = new DataStoreKeyDto
{
PartitionId = partition_id,
ObjectId = object_id
};
var object_value = new DataStoreValueDto
{
Val = value
};
Console.WriteLine(">>> Write request...");
try
{
reply = client.Write(new WriteRequest {
ObjectKey = object_key, Object = object_value
});
Console.WriteLine("Write result: " + reply);
}
catch
{
bool canRetryOperation = HandleCrashedServer(attached_server_id);
if (canRetryOperation)
{
Console.WriteLine(">>> Retrying <Write> after reattaching to new master");
write(partition_id, object_id, value);
}
return;
}
}
private bool HandleCrashedServer(string crashed_server_id)
{
Console.WriteLine("--------------------");
Console.WriteLine(">>> CLIENT: Notify Crash... " + crashed_server_id);
Console.WriteLine(">>> The server is not responding. It seems to have crashed. ServerID: " + crashed_server_id);
Console.WriteLine("--------------------");
if (NotifyPartitionsAboutCrashedServerAndReattachNewMaster(crashed_server_id))
{
PartitionMapping.RemoveCrashedServerFromAllPartitions(crashed_server_id);
ServerUrlMapping.RemoveCrashedServer(crashed_server_id);
Console.WriteLine("--------------------");
return(true);
}
return(false);
}
private static void RunTryGetInitializationInfo(
IReadOnlyDictionary <PartitionKeyRange, IPartitionedToken> expectedMappingLeftPartitions,
IReadOnlyDictionary <PartitionKeyRange, IPartitionedToken> expectedMappingTargetPartition,
IReadOnlyDictionary <PartitionKeyRange, IPartitionedToken> expectedMappingRightPartitions,
IEnumerable <PartitionKeyRange> partitionKeyRanges,
IEnumerable <IPartitionedToken> partitionedTokens)
{
TryCatch <PartitionMapping <IPartitionedToken> > tryGetInitializationInfo = PartitionMapper.MonadicGetPartitionMapping <IPartitionedToken>(
partitionKeyRanges.OrderBy(x => Guid.NewGuid()).ToArray(),
partitionedTokens.OrderBy(x => Guid.NewGuid()).ToList());
Assert.IsTrue(tryGetInitializationInfo.Succeeded);
PartitionMapping <IPartitionedToken> partitionMapping = tryGetInitializationInfo.Result;
AssertPartitionMappingAreEqual(expectedMappingLeftPartitions, partitionMapping.PartitionsLeftOfTarget);
AssertPartitionMappingAreEqual(expectedMappingTargetPartition, partitionMapping.TargetPartition);
AssertPartitionMappingAreEqual(expectedMappingRightPartitions, partitionMapping.PartitionsRightOfTarget);
}
private void updateConnectionToReplicas(string partition_id)
{
string[] replicas = PartitionMapping.GetPartitionReplicas(partition_id);
foreach (string replica in replicas)
{
GrpcChannel channel;
if (replica_channels.TryGetValue(replica, out channel))
{
channel.ShutdownAsync();
}
string url = ServerUrlMapping.GetServerUrl(replica);
channel = GrpcChannel.ForAddress(url);
replica_channels[replica] = channel;
replica_clients[replica] = new ServerCommunicationService.ServerCommunicationServiceClient(channel);
}
}
public string getMasterID()
{
return(PartitionMapping.GetPartitionMaster(this.id));
}
public void UpdatePartitionsContext(Dictionary <string, string> partitionToReplicationFactorMapping, Dictionary <string, string[]> partitionMapping)
{
PartitionMapping.CreatePartitionMapping(partitionToReplicationFactorMapping, partitionMapping);
}
public void setNewPartitionMaster(string partition_id, string new_master_id)
{
// removeing partition master means deleting the old one and assigning the new one as a master
PartitionMapping.RemovePartitionMaster(partition_id);
PartitionMapping.SetPartitionMaster(partition_id, new_master_id);
}
private static TryCatch <CrossFeedRangeState <QueryState> > MonadicExtractState(
CosmosElement continuationToken,
IReadOnlyList <FeedRangeEpk> ranges)
{
if (continuationToken == null)
{
// Full fan out to the ranges with null continuations
CrossFeedRangeState <QueryState> fullFanOutState = new CrossFeedRangeState <QueryState>(ranges.Select(range => new FeedRangeState <QueryState>(range, (QueryState)null)).ToArray());
return(TryCatch <CrossFeedRangeState <QueryState> > .FromResult(fullFanOutState));
}
if (!(continuationToken is CosmosArray parallelContinuationTokenListRaw))
{
return(TryCatch <CrossFeedRangeState <QueryState> > .FromException(
new MalformedContinuationTokenException(
$"Invalid format for continuation token {continuationToken} for {nameof(ParallelCrossPartitionQueryPipelineStage)}")));
}
if (parallelContinuationTokenListRaw.Count == 0)
{
return(TryCatch <CrossFeedRangeState <QueryState> > .FromException(
new MalformedContinuationTokenException(
$"Invalid format for continuation token {continuationToken} for {nameof(ParallelCrossPartitionQueryPipelineStage)}")));
}
List <ParallelContinuationToken> parallelContinuationTokens = new List <ParallelContinuationToken>();
foreach (CosmosElement parallelContinuationTokenRaw in parallelContinuationTokenListRaw)
{
TryCatch <ParallelContinuationToken> tryCreateParallelContinuationToken = ParallelContinuationToken.TryCreateFromCosmosElement(parallelContinuationTokenRaw);
if (tryCreateParallelContinuationToken.Failed)
{
return(TryCatch <CrossFeedRangeState <QueryState> > .FromException(
tryCreateParallelContinuationToken.Exception));
}
parallelContinuationTokens.Add(tryCreateParallelContinuationToken.Result);
}
TryCatch <PartitionMapping <ParallelContinuationToken> > partitionMappingMonad = PartitionMapper.MonadicGetPartitionMapping(
ranges,
parallelContinuationTokens);
if (partitionMappingMonad.Failed)
{
return(TryCatch <CrossFeedRangeState <QueryState> > .FromException(
partitionMappingMonad.Exception));
}
PartitionMapping <ParallelContinuationToken> partitionMapping = partitionMappingMonad.Result;
List <FeedRangeState <QueryState> > feedRangeStates = new List <FeedRangeState <QueryState> >();
List <IReadOnlyDictionary <FeedRangeEpk, ParallelContinuationToken> > rangesToInitialize = new List <IReadOnlyDictionary <FeedRangeEpk, ParallelContinuationToken> >()
{
// Skip all the partitions left of the target range, since they have already been drained fully.
partitionMapping.TargetMapping,
partitionMapping.MappingRightOfTarget,
};
foreach (IReadOnlyDictionary <FeedRangeEpk, ParallelContinuationToken> rangeToInitalize in rangesToInitialize)
{
foreach (KeyValuePair <FeedRangeEpk, ParallelContinuationToken> kvp in rangeToInitalize)
{
FeedRangeState <QueryState> feedRangeState = new FeedRangeState <QueryState>(kvp.Key, kvp.Value?.Token != null ? new QueryState(CosmosString.Create(kvp.Value.Token)) : null);
feedRangeStates.Add(feedRangeState);
}
}
CrossFeedRangeState <QueryState> crossPartitionState = new CrossFeedRangeState <QueryState>(feedRangeStates.ToArray());
return(TryCatch <CrossFeedRangeState <QueryState> > .FromResult(crossPartitionState));
}
public void CreatePartition(string replicationFactor, string partitionName, string[] serverIds)
{
PartitionMapping.CreatePartition(replicationFactor, partitionName, serverIds);
}
public void UpdateReplicationFactor(string replicationFactor)
{
PartitionMapping.UpdateReplicationFactor(replicationFactor);
}
/// <summary>
/// Saves list of entries in data source.
/// </summary>
/// <param name="logs"></param>
public void SaveAll(List<QLogEntry> logs)
{
CloudStorageAccount storageAccount = GetStorageAccount();
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
List<AzureTableMapping> tablesMapping = new List<AzureTableMapping>();
foreach (var log in logs)
{
string tableName = GetTableName(log);
AzureTableMapping tableMapping = tablesMapping.FirstOrDefault(x => x.TableName == tableName);
if (tableMapping == null)
{
tableMapping = new AzureTableMapping() { TableName = tableName };
tablesMapping.Add(tableMapping);
}
string partitionKey = log.CreatedOn.ToString("HH");
PartitionMapping partitionMapping = tableMapping.PartitionMappings.FirstOrDefault(x => x.PartitionKey == partitionKey);
if (partitionMapping == null)
{
partitionMapping = new PartitionMapping() { PartitionKey = partitionKey };
tableMapping.PartitionMappings.Add(partitionMapping);
}
partitionMapping.Logs.Add(log);
}
foreach (var tableMapping in tablesMapping)
{
CloudTable table = tableClient.GetTableReference(tableMapping.TableName);
table.CreateIfNotExists();
foreach (var partitionMapping in tableMapping.PartitionMappings)
{
List<QLogEntry> partitionLogs = partitionMapping.Logs;
//For now (23.03.2013) single batch operation may consist of at most 100 entities
//so there is need to perform "paging" in case when there are more than 100 logs for single partition
int noPages = (int)Math.Ceiling((double)partitionLogs.Count / (double)BATCH_INSERT_LIMIT);
for (int i = 0; i < noPages; i++)
{
var batchLogs = partitionLogs.Skip(BATCH_INSERT_LIMIT * i).Take(BATCH_INSERT_LIMIT);
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (var log in batchLogs)
{
log.PartitionKey = partitionMapping.PartitionKey;
log.RowKey = String.Format("{0}{1}", (DateTime.MaxValue - log.CreatedOn).Ticks.ToString("d19"), log.Guid.ToString("N"));
log.Message = log.Message.Trim();
batchOperation.Insert(log);
}
table.ExecuteBatch(batchOperation);
}
}
}
}
private void read(string partition_id, string object_id, string server_id)
{
string result = "N/A";
bool got_result = false;
ReadReply reply;
var object_key = new DataStoreKeyDto
{
PartitionId = partition_id,
ObjectId = object_id
};
if (debug_console)
{
Console.WriteLine("Reading from the server...");
}
Console.WriteLine(">>> Read request...");
// if the client is attached to a server and that server contains the desired partition
List <string> available_partitions_in_server = PartitionMapping.getPartitionsByServerID(attached_server_id);
if (!string.IsNullOrEmpty(attached_server_id) && available_partitions_in_server.Contains(partition_id))
{
if (debug_console)
{
Console.WriteLine("Reading from the Attached Server: " + attached_server_id);
}
// read value from attached server
try
{
reply = client.Read(new ReadRequest {
ObjectKey = object_key
});
if (reply.ObjectExists)
{
result = reply.Object.Val;
got_result = true;
}
// server is crashed
} catch
{
handle_crashed_server(attached_server_id);
got_result = false;
}
}
// if theres no result yet and there is a valid server_id parameter
if ((!got_result) && (!server_id.Equals("-1")))
{
// check if the server hint even has the partition
available_partitions_in_server = PartitionMapping.getPartitionsByServerID(server_id);
if (available_partitions_in_server.Contains(partition_id))
{
if (debug_console)
{
Console.WriteLine("Attach to new Server: " + server_id);
}
reattachServer(server_id);
// read value from alternative server
try
{
reply = client.Read(new ReadRequest {
ObjectKey = object_key
});
if (reply.ObjectExists)
{
result = reply.Object.Val;
got_result = true;
}
// server is crashed
} catch
{
handle_crashed_server(attached_server_id);
got_result = false;
}
}
}
// if theres no result yet, the client should find a server serving partition_id on its own
// it will try to connect to every single node in that partition
if (!got_result)
{
string[] partition_nodes = PartitionMapping.getPartitionAllNodes(partition_id);
foreach (string node_id in partition_nodes)
{
if (debug_console)
{
Console.WriteLine("Attach to new Server: " + node_id);
}
reattachServer(node_id);
// read value from one of the partition servers
try
{
reply = client.Read(new ReadRequest {
ObjectKey = object_key
});
if (reply.ObjectExists)
{
result = reply.Object.Val;
got_result = true;
}
// server is crashed
} catch
{
handle_crashed_server(attached_server_id);
got_result = false;
}
}
}
if (got_result == false)
{
result = "N/A";
}
Console.WriteLine("Read Result: " + result);
}
public void dealWithServerCrash(string partition_id, string server_id)
{
PartitionMapping.removePartitionMaster(partition_id);
}
