public NotifyCrashReply NotifyCrashHandler(NotifyCrashRequest request) { lock (_syncRoot) { string partitionName = request.PartitionId; string crashedMasterServerId = request.CrashedMasterServerId; string currentMasterServerId = PartitionMapping.GetPartitionMaster(partitionName); Console.WriteLine(">>> Received Message from the client about crashed server: PartitionName=" + partitionName + ", CrashedServerId=" + crashedMasterServerId); // check if the crashed server has been dealt with already string[] serversOfThePartition = PartitionMapping.partitionMapping[partitionName]; if (!serversOfThePartition.Contains(crashedMasterServerId)) { // this means an election already happened and the crashed server was deleted. the current partition master is the election result // another possibility is that the master detected a replica failure. In this case the partition master didn't actually change saying that it did no harm. Console.WriteLine(">>> Election Process Has Already Happened, CurrentMasterServerId=" + currentMasterServerId); return(new NotifyCrashReply { Status = "OK", MasterId = currentMasterServerId }); } // confirm that server is actually crashed SendValueToReplica svr = new SendValueToReplica(server); return(svr.HandleServerCrash(partitionName, crashedMasterServerId)); } }
private bool NotifyPartitionsAboutCrashedServerAndReattachNewMaster(string crashedServerId) { List <string> partitions = PartitionMapping.GetPartitionsThatContainServer(crashedServerId); foreach (string partitionName in partitions) { Console.WriteLine(">>> Notify Servers in Partition about Crash..."); Console.WriteLine(">>> Partition=" + partitionName); string[] partitionNodes = PartitionMapping.GetPartitionAllNodes(partitionName); bool newMasterWasReattachedSuccessfully = false; foreach (string serverId in partitionNodes) { if (serverId == crashedServerId) { continue; // Skip crashed server } else if (TryNotifyServerAboutCrashedServer(partitionName, serverId, crashedServerId)) { newMasterWasReattachedSuccessfully = true; break; } } return(newMasterWasReattachedSuccessfully); } return(false); }
private bool TryNotifyServerAboutCrashedServer(string partitionName, string serverId, string crashedServerId) { Console.WriteLine(">>> Notifying Server=" + serverId + " about CrashedServer=" + crashedServerId); try { reattachServer(serverId); NotifyCrashReply notifyCrashReply = client.NotifyCrash(new NotifyCrashRequest { PartitionId = partitionName, CrashedMasterServerId = crashedServerId }); Console.WriteLine(">>> Got Reply from ServerId=" + serverId); Console.WriteLine(">>> New Partition Master: PartitionName=" + partitionName + "PartitionMasterId=" + notifyCrashReply.MasterId); string masterId = notifyCrashReply.MasterId; PartitionMapping.SetPartitionMaster(partitionName, masterId); if (serverId != masterId) { Console.WriteLine(">>> Reataching to new master. MasterId=" + masterId); reattachServer(masterId); } else { Console.WriteLine(">>> Already Attached to new master. MasterId=" + masterId); } return(true); } catch { Console.WriteLine(">>> No Reply..."); return(false); } }
// Change name to reflect boolean return private bool CompareClock(string partition_id, int reply_clock, DataStoreKeyDto object_key) { // Do Clock related staff int partition_highest_clock = PartitionMapping.partitionToClockMapping[partition_id]; Console.WriteLine(">>> PartitionClock=" + partition_highest_clock + ", ReplyClock=" + reply_clock); if (reply_clock > partition_highest_clock) { Console.WriteLine(">>> ReplyClock > PartitionClock. Reply clock is acceptable. "); PartitionMapping.UpdatePartitionClock(partition_id, reply_clock); return(true); } else if (reply_clock < partition_highest_clock) { Console.WriteLine(">>> ReplyClock < PartitionClock. Waiting " + retry_time.ToString() + "And trying again..."); wait(retry_time); return(TryReadValue(object_key, partition_id)); } else if (reply_clock == partition_highest_clock) { Console.WriteLine(">>> PartitionClock == ReplyClock. Reply clock is acceptable. "); } return(true); }
public void createPartition(string partition_id) { String master_id = PartitionMapping.getPartitionMaster(partition_id); bool is_master = master_id.Equals(server_id); Partition p = new Partition(partition_id, is_master); partitions.Add(p); }
public void CreateLocalPartitions() { List <string> partitions = PartitionMapping.GetPartitionsByServerID(serverId); foreach (string partition_id in partitions) { server.createPartition(partition_id); } }
public void createPartition(string partition_id) { string master_id = PartitionMapping.GetPartitionMaster(partition_id); bool is_master = master_id.Equals(server_id); int partition_clock = PartitionMapping.GetPartitionClock(partition_id); Partition p = new Partition(partition_id, is_master, partition_clock); partitions.Add(p); }
private void write(string partition_id, string object_id, string value) { WriteReply reply; if (debug_console) { Console.WriteLine("Get Partition Master from Partition Named: " + partition_id); } string partition_master_server_id = PartitionMapping.GetPartitionMaster(partition_id); if (debug_console) { Console.WriteLine("Partition Master Server ID: " + partition_master_server_id); } reattachServer(partition_master_server_id); var object_key = new DataStoreKeyDto { PartitionId = partition_id, ObjectId = object_id }; var object_value = new DataStoreValueDto { Val = value }; Console.WriteLine(">>> Write request..."); try { reply = client.Write(new WriteRequest { ObjectKey = object_key, Object = object_value }); Console.WriteLine("Write result: " + reply); } catch { bool canRetryOperation = HandleCrashedServer(attached_server_id); if (canRetryOperation) { Console.WriteLine(">>> Retrying <Write> after reattaching to new master"); write(partition_id, object_id, value); } return; } }
private bool HandleCrashedServer(string crashed_server_id) { Console.WriteLine("--------------------"); Console.WriteLine(">>> CLIENT: Notify Crash... " + crashed_server_id); Console.WriteLine(">>> The server is not responding. It seems to have crashed. ServerID: " + crashed_server_id); Console.WriteLine("--------------------"); if (NotifyPartitionsAboutCrashedServerAndReattachNewMaster(crashed_server_id)) { PartitionMapping.RemoveCrashedServerFromAllPartitions(crashed_server_id); ServerUrlMapping.RemoveCrashedServer(crashed_server_id); Console.WriteLine("--------------------"); return(true); } return(false); }
private static void RunTryGetInitializationInfo( IReadOnlyDictionary <PartitionKeyRange, IPartitionedToken> expectedMappingLeftPartitions, IReadOnlyDictionary <PartitionKeyRange, IPartitionedToken> expectedMappingTargetPartition, IReadOnlyDictionary <PartitionKeyRange, IPartitionedToken> expectedMappingRightPartitions, IEnumerable <PartitionKeyRange> partitionKeyRanges, IEnumerable <IPartitionedToken> partitionedTokens) { TryCatch <PartitionMapping <IPartitionedToken> > tryGetInitializationInfo = PartitionMapper.MonadicGetPartitionMapping <IPartitionedToken>( partitionKeyRanges.OrderBy(x => Guid.NewGuid()).ToArray(), partitionedTokens.OrderBy(x => Guid.NewGuid()).ToList()); Assert.IsTrue(tryGetInitializationInfo.Succeeded); PartitionMapping <IPartitionedToken> partitionMapping = tryGetInitializationInfo.Result; AssertPartitionMappingAreEqual(expectedMappingLeftPartitions, partitionMapping.PartitionsLeftOfTarget); AssertPartitionMappingAreEqual(expectedMappingTargetPartition, partitionMapping.TargetPartition); AssertPartitionMappingAreEqual(expectedMappingRightPartitions, partitionMapping.PartitionsRightOfTarget); }
private void updateConnectionToReplicas(string partition_id) { string[] replicas = PartitionMapping.GetPartitionReplicas(partition_id); foreach (string replica in replicas) { GrpcChannel channel; if (replica_channels.TryGetValue(replica, out channel)) { channel.ShutdownAsync(); } string url = ServerUrlMapping.GetServerUrl(replica); channel = GrpcChannel.ForAddress(url); replica_channels[replica] = channel; replica_clients[replica] = new ServerCommunicationService.ServerCommunicationServiceClient(channel); } }
public string getMasterID() { return(PartitionMapping.GetPartitionMaster(this.id)); }
public void UpdatePartitionsContext(Dictionary <string, string> partitionToReplicationFactorMapping, Dictionary <string, string[]> partitionMapping) { PartitionMapping.CreatePartitionMapping(partitionToReplicationFactorMapping, partitionMapping); }
public void setNewPartitionMaster(string partition_id, string new_master_id) { // removeing partition master means deleting the old one and assigning the new one as a master PartitionMapping.RemovePartitionMaster(partition_id); PartitionMapping.SetPartitionMaster(partition_id, new_master_id); }
private static TryCatch <CrossFeedRangeState <QueryState> > MonadicExtractState( CosmosElement continuationToken, IReadOnlyList <FeedRangeEpk> ranges) { if (continuationToken == null) { // Full fan out to the ranges with null continuations CrossFeedRangeState <QueryState> fullFanOutState = new CrossFeedRangeState <QueryState>(ranges.Select(range => new FeedRangeState <QueryState>(range, (QueryState)null)).ToArray()); return(TryCatch <CrossFeedRangeState <QueryState> > .FromResult(fullFanOutState)); } if (!(continuationToken is CosmosArray parallelContinuationTokenListRaw)) { return(TryCatch <CrossFeedRangeState <QueryState> > .FromException( new MalformedContinuationTokenException( $"Invalid format for continuation token {continuationToken} for {nameof(ParallelCrossPartitionQueryPipelineStage)}"))); } if (parallelContinuationTokenListRaw.Count == 0) { return(TryCatch <CrossFeedRangeState <QueryState> > .FromException( new MalformedContinuationTokenException( $"Invalid format for continuation token {continuationToken} for {nameof(ParallelCrossPartitionQueryPipelineStage)}"))); } List <ParallelContinuationToken> parallelContinuationTokens = new List <ParallelContinuationToken>(); foreach (CosmosElement parallelContinuationTokenRaw in parallelContinuationTokenListRaw) { TryCatch <ParallelContinuationToken> tryCreateParallelContinuationToken = ParallelContinuationToken.TryCreateFromCosmosElement(parallelContinuationTokenRaw); if (tryCreateParallelContinuationToken.Failed) { return(TryCatch <CrossFeedRangeState <QueryState> > .FromException( tryCreateParallelContinuationToken.Exception)); } parallelContinuationTokens.Add(tryCreateParallelContinuationToken.Result); } TryCatch <PartitionMapping <ParallelContinuationToken> > partitionMappingMonad = PartitionMapper.MonadicGetPartitionMapping( ranges, parallelContinuationTokens); if (partitionMappingMonad.Failed) { return(TryCatch <CrossFeedRangeState <QueryState> > .FromException( partitionMappingMonad.Exception)); } PartitionMapping <ParallelContinuationToken> partitionMapping = partitionMappingMonad.Result; List <FeedRangeState <QueryState> > feedRangeStates = new List <FeedRangeState <QueryState> >(); List <IReadOnlyDictionary <FeedRangeEpk, ParallelContinuationToken> > rangesToInitialize = new List <IReadOnlyDictionary <FeedRangeEpk, ParallelContinuationToken> >() { // Skip all the partitions left of the target range, since they have already been drained fully. partitionMapping.TargetMapping, partitionMapping.MappingRightOfTarget, }; foreach (IReadOnlyDictionary <FeedRangeEpk, ParallelContinuationToken> rangeToInitalize in rangesToInitialize) { foreach (KeyValuePair <FeedRangeEpk, ParallelContinuationToken> kvp in rangeToInitalize) { FeedRangeState <QueryState> feedRangeState = new FeedRangeState <QueryState>(kvp.Key, kvp.Value?.Token != null ? new QueryState(CosmosString.Create(kvp.Value.Token)) : null); feedRangeStates.Add(feedRangeState); } } CrossFeedRangeState <QueryState> crossPartitionState = new CrossFeedRangeState <QueryState>(feedRangeStates.ToArray()); return(TryCatch <CrossFeedRangeState <QueryState> > .FromResult(crossPartitionState)); }
public void CreatePartition(string replicationFactor, string partitionName, string[] serverIds) { PartitionMapping.CreatePartition(replicationFactor, partitionName, serverIds); }
public void UpdateReplicationFactor(string replicationFactor) { PartitionMapping.UpdateReplicationFactor(replicationFactor); }
/// <summary> /// Saves list of entries in data source. /// </summary> /// <param name="logs"></param> public void SaveAll(List<QLogEntry> logs) { CloudStorageAccount storageAccount = GetStorageAccount(); CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); List<AzureTableMapping> tablesMapping = new List<AzureTableMapping>(); foreach (var log in logs) { string tableName = GetTableName(log); AzureTableMapping tableMapping = tablesMapping.FirstOrDefault(x => x.TableName == tableName); if (tableMapping == null) { tableMapping = new AzureTableMapping() { TableName = tableName }; tablesMapping.Add(tableMapping); } string partitionKey = log.CreatedOn.ToString("HH"); PartitionMapping partitionMapping = tableMapping.PartitionMappings.FirstOrDefault(x => x.PartitionKey == partitionKey); if (partitionMapping == null) { partitionMapping = new PartitionMapping() { PartitionKey = partitionKey }; tableMapping.PartitionMappings.Add(partitionMapping); } partitionMapping.Logs.Add(log); } foreach (var tableMapping in tablesMapping) { CloudTable table = tableClient.GetTableReference(tableMapping.TableName); table.CreateIfNotExists(); foreach (var partitionMapping in tableMapping.PartitionMappings) { List<QLogEntry> partitionLogs = partitionMapping.Logs; //For now (23.03.2013) single batch operation may consist of at most 100 entities //so there is need to perform "paging" in case when there are more than 100 logs for single partition int noPages = (int)Math.Ceiling((double)partitionLogs.Count / (double)BATCH_INSERT_LIMIT); for (int i = 0; i < noPages; i++) { var batchLogs = partitionLogs.Skip(BATCH_INSERT_LIMIT * i).Take(BATCH_INSERT_LIMIT); TableBatchOperation batchOperation = new TableBatchOperation(); foreach (var log in batchLogs) { log.PartitionKey = partitionMapping.PartitionKey; log.RowKey = String.Format("{0}{1}", (DateTime.MaxValue - log.CreatedOn).Ticks.ToString("d19"), log.Guid.ToString("N")); log.Message = log.Message.Trim(); batchOperation.Insert(log); } table.ExecuteBatch(batchOperation); } } } }
private void read(string partition_id, string object_id, string server_id) { string result = "N/A"; bool got_result = false; ReadReply reply; var object_key = new DataStoreKeyDto { PartitionId = partition_id, ObjectId = object_id }; if (debug_console) { Console.WriteLine("Reading from the server..."); } Console.WriteLine(">>> Read request..."); // if the client is attached to a server and that server contains the desired partition List <string> available_partitions_in_server = PartitionMapping.getPartitionsByServerID(attached_server_id); if (!string.IsNullOrEmpty(attached_server_id) && available_partitions_in_server.Contains(partition_id)) { if (debug_console) { Console.WriteLine("Reading from the Attached Server: " + attached_server_id); } // read value from attached server try { reply = client.Read(new ReadRequest { ObjectKey = object_key }); if (reply.ObjectExists) { result = reply.Object.Val; got_result = true; } // server is crashed } catch { handle_crashed_server(attached_server_id); got_result = false; } } // if theres no result yet and there is a valid server_id parameter if ((!got_result) && (!server_id.Equals("-1"))) { // check if the server hint even has the partition available_partitions_in_server = PartitionMapping.getPartitionsByServerID(server_id); if (available_partitions_in_server.Contains(partition_id)) { if (debug_console) { Console.WriteLine("Attach to new Server: " + server_id); } reattachServer(server_id); // read value from alternative server try { reply = client.Read(new ReadRequest { ObjectKey = object_key }); if (reply.ObjectExists) { result = reply.Object.Val; got_result = true; } // server is crashed } catch { handle_crashed_server(attached_server_id); got_result = false; } } } // if theres no result yet, the client should find a server serving partition_id on its own // it will try to connect to every single node in that partition if (!got_result) { string[] partition_nodes = PartitionMapping.getPartitionAllNodes(partition_id); foreach (string node_id in partition_nodes) { if (debug_console) { Console.WriteLine("Attach to new Server: " + node_id); } reattachServer(node_id); // read value from one of the partition servers try { reply = client.Read(new ReadRequest { ObjectKey = object_key }); if (reply.ObjectExists) { result = reply.Object.Val; got_result = true; } // server is crashed } catch { handle_crashed_server(attached_server_id); got_result = false; } } } if (got_result == false) { result = "N/A"; } Console.WriteLine("Read Result: " + result); }
public void dealWithServerCrash(string partition_id, string server_id) { PartitionMapping.removePartitionMaster(partition_id); }