public async Task <Task> Commit(StreamMessage msg) { List <TopologyUnit> units = topology.GetAllTopologyUnits(); PrettyConsole.Line("Number of units: " + units.Count); msg.barrierOrCommitInfo = new BarrierOrCommitMsgTrackingInfo(Guid.NewGuid(), units.Count); msg.barrierOrCommitInfo.BatchID = msg.BatchID; await batchTracker.TrackingCommitMessages(msg); foreach (TopologyUnit unit in units) { if (unit.OperatorType == OperatorType.Source) { IStreamSource source = GrainFactory.GetGrain <IStreamSource>(unit.PrimaryKey); source.Commit(msg); } else if (unit.OperatorType == OperatorType.Stateful) { IStatefulOperator statefulOperator = GrainFactory.GetGrain <IStatefulOperator>(unit.PrimaryKey, Constants.Stateful_Operator_Prefix); statefulOperator.Commit(msg); } else if (unit.OperatorType == OperatorType.Stateless) { IStatelessOperator statelessOperator = GrainFactory.GetGrain <IStatelessOperator>(unit.PrimaryKey, Constants.Stateless_Operator_Prefix); statelessOperator.Commit(msg); } else { throw new ArgumentException("Commit: The operator type is in valid!"); } } return(Task.CompletedTask); }
//Tracking the commit messages. public Task TrackingCommitMessages(StreamMessage msg) { if (commitTrackingMap.ContainsKey(msg.BatchID)) { var targetBatch = commitTrackingMap[msg.BatchID]; Functions.CheckNotNull(msg.barrierOrCommitInfo); targetBatch.AddBarrierOrCommitMsgTrackingHelper(msg.barrierOrCommitInfo); } else if (msg.BatchID >= 0 && !committedBatch.Contains(msg.BatchID)) { PrettyConsole.Line("Committing a new batch" + msg.BatchID); StreamBatch newBatch = new StreamBatch(msg.BatchID); //The name should be changed Functions.CheckNotNull(msg.barrierOrCommitInfo); newBatch.AddBarrierOrCommitMsgTrackingHelper(msg.barrierOrCommitInfo); commitTrackingMap.Add(msg.BatchID, newBatch); //PrettyConsole.Line("Add batch " + msg.BatchID + " to the commitTrackingMap"); } else { throw new InvalidOperationException("Batch ID less 0 or the message of that batch ahs been committed"); } return(Task.CompletedTask); }
//Recovery public async Task <Task> CompleteOneOperatorRecovery(BarrierOrCommitMsgTrackingInfo msgInfo) { if (!recoveryTrackingMap.ContainsKey(msgInfo.BatchID)) { //Multiple batch has that problem PrettyConsole.Line("The recovery key " + msgInfo.BatchID + " is not exist"); } else { //PrettyConsole.Line("Finish Tracking one message in batchID: " + msgInfo.BatchID); StreamBatch targetBatch = recoveryTrackingMap[msgInfo.BatchID]; targetBatch.CompleteOneMessageTracking(msgInfo); if (targetBatch.readForCommitting) { if (batchCoordinator != null) { PrettyConsole.Line("Batch: " + msgInfo.BatchID + " commit has been successfully recoveryed"); await batchCoordinator.CompleteRecovery(msgInfo.BatchID); recoveryTrackingMap.Remove(msgInfo.BatchID); } } } return(Task.CompletedTask); }
public Task MarkOperatorAsFailed() { isOperatorFailed = true; isARestartOperator = true; PrettyConsole.Line("Mark this as failed!"); return(Task.CompletedTask); }
public Task RemoveCustomDownStreamOperator(Guid guid) { int index = -1; for (int i = 0; i < downStreamOperators.Count; i++) { if (downStreamOperators[i].GetPrimaryKey() == guid) { index = i; break; } } if (index != -1) { downStreamOperators.RemoveAt(index); PrettyConsole.Line("Remove old stateful from upper stream"); operatorSettings.RemoveOperatorFromDict(guid); topologyManager.UpdateOperatorSettings(this.GetPrimaryKey(), operatorSettings); } else { throw new ArgumentException(); } return(Task.CompletedTask); }
private bool CheckCount(StreamMessage msg) { if (!upStreamMessageCountMaps.ContainsKey(msg.BatchID)) { if (msg.Count == 0) { return(true); } else { return(false); } } else if (upStreamMessageCountMaps[msg.BatchID].ContainsKey(msg.From) && msg.Count == upStreamMessageCountMaps[msg.BatchID][msg.From]) { return(true); } else if (!upStreamMessageCountMaps[msg.BatchID].ContainsKey(msg.From) && msg.Count == 0) { return(true); } else { PrettyConsole.Line("The count in stateless operator is not equal!"); return(false); } }
//Commit public Task StartCommit(int ID) { PrettyConsole.Line("Start Commit Batch " + ID); commitMsg.BatchID = ID; topologyManager.Commit(commitMsg); return(Task.CompletedTask); }
public override Task OnNextAsync(ChatMsg item, StreamSequenceToken token = null) { if (item.Author == "System") { PrettyConsole.Line($"==== System message: '{item.Text}'", ConsoleColor.Green); } return(Task.CompletedTask); }
private async Task <Task> DetectPossibleFailures(object org) { PrettyConsole.Line("Start Detect Failures"); Dictionary <Guid, Task <int> > taskMap = new Dictionary <Guid, Task <int> >(); List <Task <int> > taskList = new List <Task <int> >(); var topologyManager = GrainFactory.GetGrain <ITopology>(Constants.Topology_Manager); var operatorUnits = await topologyManager.GetAllUnits(); foreach (TopologyUnit unit in operatorUnits) { if (unit.OperatorType == OperatorType.Source) { IStreamSource source = GrainFactory.GetGrain <IStreamSource>(unit.PrimaryKey); var task = source.DetectErrors(); taskMap.Add(unit.PrimaryKey, task); } else if (unit.OperatorType == OperatorType.Stateful) { IStatefulOperator statefulOperator = GrainFactory.GetGrain <IStatefulOperator>(unit.PrimaryKey, Constants.Stateful_Operator_Prefix); var task = statefulOperator.DetectErrors(); taskMap.Add(unit.PrimaryKey, task); } else if (unit.OperatorType == OperatorType.Stateless) { IStatelessOperator statelessOperator = GrainFactory.GetGrain <IStatelessOperator>(unit.PrimaryKey, Constants.Stateless_Operator_Prefix); var task = statelessOperator.DetectErrors(); taskMap.Add(unit.PrimaryKey, task); } else { throw new ArgumentException("Commit: The operator type is in valid!"); } } try { await Task.WhenAny(Task.WhenAll(taskMap.Values), Task.Delay(TimeSpan.FromSeconds(2))); } catch (Exception e) { PrettyConsole.Line(e.ToString()); } foreach (var task in taskMap) { if (task.Value.Status != TaskStatus.RanToCompletion) { PrettyConsole.Line("Replace!"); await topologyManager.ReplaceTheOldOperator(task.Key); disposable.Dispose(); break; } } return(Task.CompletedTask); }
private Task TellTrackMessageSent(StreamMessage item) { if (tracker != null) { tracker.CompleteTracking(item); PrettyConsole.Line("Complete one barrier"); } return(Task.CompletedTask); }
//Replay Logic public async Task <Task> ReplayTheMessageOnRecoveryCompleted() { PrettyConsole.Line("Start Replay!"); foreach (StreamMessage msg in messagesForRecovery) { await ProcessNormalMessage(msg); } return(Task.CompletedTask); }
protected async Task <Task> ExecuteMessagesByDownStreamOperators(StreamMessage msg, IAsyncStream <StreamMessage> stream, IOperator op) { if (downStreamOperators.Count > 0) { int batchID = msg.BatchID; var targetKey = op.GetPrimaryKey(); try { msg.From = this.GetPrimaryKey(); //if is barrier message, set the message count if (msg.Value == Constants.Barrier_Value) { if (downStreamMessageCountMaps.ContainsKey(batchID)) { if (downStreamMessageCountMaps[batchID].ContainsKey(op.GetPrimaryKey())) { msg.Count = downStreamMessageCountMaps[batchID][op.GetPrimaryKey()]; } else { msg.Count = 0; } } else { msg.Count = 0; } } //if it is a normal message, increment the count map else if (msg.Value != Constants.System_Key) { if (!downStreamMessageCountMaps.ContainsKey(batchID)) { downStreamMessageCountMaps.Add(batchID, new Dictionary <Guid, int>()); } var key = op.GetPrimaryKey(); if (downStreamMessageCountMaps[batchID].ContainsKey(key)) { downStreamMessageCountMaps[batchID][key] = downStreamMessageCountMaps[batchID][key] + 1; } else { downStreamMessageCountMaps[batchID].Add(key, 1); } } op.ExecuteMessage(msg, stream); } catch (Exception e) { PrettyConsole.Line("Get Exception : " + e + "; Start Receovry"); topologyManager.ReplaceTheOldOperator(targetKey); } } return(Task.CompletedTask); }
public override Task OnNextAsync(ChatMsg item, StreamSequenceToken token = null) { var textHasMention = item.Text.Contains("@"); if (textHasMention) { PrettyConsole.Line($"==== MENTION DETECTED: '{item.Author}' mentions someone. Sending a notification email.", ConsoleColor.Green); } return(Task.CompletedTask); }
public override Task OnActivateAsync() { currentBatchID = 0; committedID = -1; tracker = GrainFactory.GetGrain <IBatchTracker>(Utils.Constants.Tracker); topologyManager = GrainFactory.GetGrain <ITopology>(Constants.Topology_Manager); PrettyConsole.Line("Register Timer"); var streamProvider = GetStreamProvider(Constants.FaultTolerantStreamProvider); return(base.OnActivateAsync()); }
private static void PrintHints() { var menuColor = ConsoleColor.Magenta; PrettyConsole.Line("Type '/j <channel>' to join specific channel", menuColor); PrettyConsole.Line("Type '/n <username>' to set your user name", menuColor); PrettyConsole.Line("Type '/l' to leave specific channel", menuColor); PrettyConsole.Line("Type '<any text>' to send a message", menuColor); PrettyConsole.Line("Type '/h' to re-read channel history", menuColor); PrettyConsole.Line("Type '/m' to query members in the channel", menuColor); PrettyConsole.Line("Type '/exit' to exit client.", menuColor); }
private static async Task ShowChannelMembers(IClusterClient client) { var room = client.GetGrain <IStreamSource>(joinedChannel); var members = await room.GetMembers(); PrettyConsole.Line($"====== Members for '{joinedChannel}' Channel ======", ConsoleColor.DarkGreen); foreach (var member in members) { PrettyConsole.Line(member, ConsoleColor.DarkGreen); } PrettyConsole.Line("============", ConsoleColor.DarkGreen); }
public static void Main(string[] args) { var clientInstance = InitializeClient(args).GetAwaiter().GetResult(); PrettyConsole.Line("==== CLIENT: Initialized ====", ConsoleColor.Cyan); PrettyConsole.Line("CLIENT: Write commands:", ConsoleColor.Cyan); Menu(clientInstance).GetAwaiter().GetResult(); PrettyConsole.Line("==== CLIENT: Shutting down ====", ConsoleColor.DarkRed); }
private static async Task ShowCurrentChannelHistory(IClusterClient client) { var room = client.GetGrain <IStreamSource>(joinedChannel); var history = await room.ReadHistory(1000); PrettyConsole.Line($"====== History for '{joinedChannel}' Channel ======", ConsoleColor.DarkGreen); foreach (var chatMsg in history) { PrettyConsole.Line($" ({chatMsg.Created:g}) {chatMsg.Key}> {chatMsg.Value}", ConsoleColor.DarkGreen); } PrettyConsole.Line("============", ConsoleColor.DarkGreen); }
//This function get the words and count public async Task <Task> ExecuteMessage(StreamMessage msg, IAsyncStream <StreamMessage> stream) { //At frist, if it is a new batch, just creat the incremental log //for it if (!incrementalLogMap.ContainsKey(msg.BatchID)) { var newIncrementalLog = new Dictionary <string, object>(); var newReverseLog = new Dictionary <string, object>(); incrementalLogMap.Add(msg.BatchID, newIncrementalLog); reverseLogMap.Add(msg.BatchID, newReverseLog); } if (msg.BatchID > currentReverseLogID) { currentReverseLogID = msg.BatchID; } if (msg.BatchID > currentBatchID) { messageBuffer.Add(msg); asyncStream = stream; } else if (msg.BatchID == currentBatchID || msg.Key == Constants.System_Key) { if (msg.Key != Constants.System_Key) { await IncrementUpStreamCount(msg); await CustomExecutionMethod(msg, stream); currentWordsProcessed++; if (!isARestartOperator && currentWordsProcessed >= numOfMaxProcessWords && isFailureTestOpen) { Thread.Sleep(10000); } } else { await ProcessSpecialMessage(msg); if (downStreamOperators.Count > 0) { await BroadcastSpecialMessage(msg, stream); } } } else { PrettyConsole.Line("ERRROOOOOOROOOR"); throw new InvalidOperationException(msg.Key + " " + msg.Value + " The id " + msg.BatchID + " is less than the currentID:" + currentBatchID); } return(Task.CompletedTask); }
public Task <bool> Tick(int exceptionMod) { counter++; PrettyConsole.Line($"Tick: {counter}"); if (counter >= 0 && counter % exceptionMod == 0) { PrettyConsole.Line("EXCEPTION !!!!", ConsoleColor.Red); throw new Exception("Bang !!!"); } return(Task.FromResult(true)); }
public async Task <object> Invoke(MethodInfo method, InvokeMethodRequest request, IGrainMethodInvoker invoker) { try { return(await invoker.Invoke(this, request)); } catch (Exception e) { PrettyConsole.Line("Exception in interceptor"); DeactivateOnIdle(); throw; } }
public Task CompleteCommit(int batchID) { if (batchID - committedID == 1) { committedID++; PrettyConsole.Line("Committed Batch ID now is: " + committedID); return(Task.CompletedTask); } else { throw new InvalidOperationException("Cannot commit batch greater than committed id 2"); } }
protected Task SaveStateToFile(IncrementalLog state) { PrettyConsole.Line("Save the incremental log to " + operatorSettings.incrementalLogAddress); try { WriteToBinaryFile(operatorSettings.incrementalLogAddress, state); } catch (Exception e) { PrettyConsole.Line("Error " + e); } return(Task.CompletedTask); }
public Task AddProcessingTime(int time) { processingTimeList.Add(time); if (processingTimeList.Count >= Processing_Time_Interval) { numOfWordsProcessed += Processing_Time_Interval; if (numOfWordsProcessed % 1000 == 0) { PrettyConsole.Line("Process " + numOfWordsProcessed + " words"); } SaveProcessingTimeIntoFiles(); processingTimeList.Clear(); } return(Task.CompletedTask); }
private static async Task <IClusterClient> InitializeClient(string[] args) { int initializeCounter = 0; var initSucceed = false; while (!initSucceed) { try { var client = new ClientBuilder().Configure <ClusterOptions>(options => { options.ClusterId = Constants.ClusterId; options.ServiceId = Constants.ServiceId; }) .UseLocalhostClustering() .ConfigureApplicationParts(parts => parts.AddApplicationPart(typeof(IStreamSource).Assembly).WithReferences()) .ConfigureLogging(logging => logging.AddConsole()) //Depends on your application requirements, you can configure your client with other stream providers, which can provide other features, //such as persistence or recoverability. For more information, please see http://dotnet.github.io/orleans/Documentation/Orleans-Streams/Stream-Providers.html .AddSimpleMessageStreamProvider(Constants.ChatRoomStreamProvider) .Build(); await client.Connect(); initSucceed = client.IsInitialized; if (initSucceed) { return(client); } } catch (Exception exc) { PrettyConsole.Line(exc.Message, ConsoleColor.Cyan); initSucceed = false; } if (initializeCounter++ > 10) { return(null); } PrettyConsole.Line("Client Init Failed. Sleeping 5s...", ConsoleColor.Red); Thread.Sleep(TimeSpan.FromSeconds(5)); } return(null); }
//If it is special message, it has to send to all the operators. //If it is barrier message, batch manager will start to track it //by using BarrierMsgTrackingInfo which keep and ID and the number of //client it sent to. private async Task <Task> ProcessSpecialMessage(StreamMessage msg, IAsyncStream <StreamMessage> stream) { BarrierOrCommitMsgTrackingInfo info = new BarrierOrCommitMsgTrackingInfo(msg.barrierOrCommitInfo.GetID(), msg.barrierOrCommitInfo.numberOfClientSent); info.BatchID = msg.BatchID; if (msg.Value == Constants.Barrier_Value && msg.BatchID - currentBatchID < 2) { currentBatchID = msg.BatchID + 1; PrettyConsole.Line("Increment ID " + currentBatchID); await TrackingBarrierMessages(msg); await BroadcastSpecialMessage(msg, stream); await batchTracker.CompleteOneOperatorBarrier(info); } return(Task.CompletedTask); }
public async Task <Task> SaveIncrementalLogIntoStorage() { //Once save the state to files, then clear //The incremental log if ((currentBatchID % Constants.Checkpoint_Interval == 0) && currentBatchID != 0) { await SaveStateToFile(new IncrementalLog(statesMap, currentBatchID, LogType.CheckPoint)); PrettyConsole.Line("Save checkpoint: " + currentBatchID % Constants.Checkpoint_Interval + " in batch: " + currentBatchID); } else { var incrementalLog = await GetIncrementalLog(currentBatchID); await SaveStateToFile(new IncrementalLog(incrementalLog, currentBatchID, LogType.Incremental)); } return(Task.CompletedTask); }
//Once the recovery completed, just restart the timer //Restart the timer public async Task <Task> CompleteRecovery(int batchID) { if (committedID == batchID) { currentBatchID = batchID + 1; disposable = RegisterTimer(SendBarrierOnPeriodOfTime, null, barrierTimeInterval, barrierTimeInterval); var detector = GrainFactory.GetGrain <IErrorDetector>(Constants.Error_Detector); detector.RegisterTimerToDetectFailures(); var recoveryTime = System.DateTime.Now.Millisecond - startTime; PrettyConsole.Line("Recovery Time: " + recoveryTime); return(Task.CompletedTask); } else { throw new InvalidOperationException("The recvoery batch is not equal to the latest committed ID"); } }
public async Task <Task> AddCustomeOperatorsToNonSourceOperators(List <IOperator> upperOps, List <IOperator> downOps) { List <TopologyUnit> units = new List <TopologyUnit>(); foreach (var op in downOps) { units.Add(await op.GetTopologyUnit()); } foreach (var op in upperOps) { op.AddCustomDownStreamOperators(units); PrettyConsole.Line("Add 1"); } return(Task.CompletedTask); }
private static async Task LeaveChannel(IClusterClient client) { PrettyConsole.Line($"Leaving channel {joinedChannel}"); var room = client.GetGrain <IStreamSource>(joinedChannel); var streamId = await room.Leave(userName); var stream = client.GetStreamProvider(Constants.ChatRoomStreamProvider) .GetStream <StreamMessage>(streamId, Constants.CharRoomStreamNameSpace); //unsubscribe from the channel/stream since client left, so that client won't //receive furture messages from this channel/stream var subscriptionHandles = await stream.GetAllSubscriptionHandles(); foreach (var handle in subscriptionHandles) { await handle.UnsubscribeAsync(); } }