public async Task <Task> Commit(StreamMessage msg)
{
List <TopologyUnit> units = topology.GetAllTopologyUnits();
PrettyConsole.Line("Number of units: " + units.Count);
msg.barrierOrCommitInfo = new BarrierOrCommitMsgTrackingInfo(Guid.NewGuid(), units.Count);
msg.barrierOrCommitInfo.BatchID = msg.BatchID;
await batchTracker.TrackingCommitMessages(msg);
foreach (TopologyUnit unit in units)
{
if (unit.OperatorType == OperatorType.Source)
{
IStreamSource source = GrainFactory.GetGrain <IStreamSource>(unit.PrimaryKey);
source.Commit(msg);
}
else if (unit.OperatorType == OperatorType.Stateful)
{
IStatefulOperator statefulOperator = GrainFactory.GetGrain <IStatefulOperator>(unit.PrimaryKey, Constants.Stateful_Operator_Prefix);
statefulOperator.Commit(msg);
}
else if (unit.OperatorType == OperatorType.Stateless)
{
IStatelessOperator statelessOperator = GrainFactory.GetGrain <IStatelessOperator>(unit.PrimaryKey, Constants.Stateless_Operator_Prefix);
statelessOperator.Commit(msg);
}
else
{
throw new ArgumentException("Commit: The operator type is in valid!");
}
}
return(Task.CompletedTask);
}
//Tracking the commit messages.
public Task TrackingCommitMessages(StreamMessage msg)
{
if (commitTrackingMap.ContainsKey(msg.BatchID))
{
var targetBatch = commitTrackingMap[msg.BatchID];
Functions.CheckNotNull(msg.barrierOrCommitInfo);
targetBatch.AddBarrierOrCommitMsgTrackingHelper(msg.barrierOrCommitInfo);
}
else if (msg.BatchID >= 0 && !committedBatch.Contains(msg.BatchID))
{
PrettyConsole.Line("Committing a new batch" + msg.BatchID);
StreamBatch newBatch = new StreamBatch(msg.BatchID);
//The name should be changed
Functions.CheckNotNull(msg.barrierOrCommitInfo);
newBatch.AddBarrierOrCommitMsgTrackingHelper(msg.barrierOrCommitInfo);
commitTrackingMap.Add(msg.BatchID, newBatch);
//PrettyConsole.Line("Add batch " + msg.BatchID + " to the commitTrackingMap");
}
else
{
throw new InvalidOperationException("Batch ID less 0 or the message of that batch ahs been committed");
}
return(Task.CompletedTask);
}
//Recovery
public async Task <Task> CompleteOneOperatorRecovery(BarrierOrCommitMsgTrackingInfo msgInfo)
{
if (!recoveryTrackingMap.ContainsKey(msgInfo.BatchID))
{
//Multiple batch has that problem
PrettyConsole.Line("The recovery key " + msgInfo.BatchID + " is not exist");
}
else
{
//PrettyConsole.Line("Finish Tracking one message in batchID: " + msgInfo.BatchID);
StreamBatch targetBatch = recoveryTrackingMap[msgInfo.BatchID];
targetBatch.CompleteOneMessageTracking(msgInfo);
if (targetBatch.readForCommitting)
{
if (batchCoordinator != null)
{
PrettyConsole.Line("Batch: " + msgInfo.BatchID + " commit has been successfully recoveryed");
await batchCoordinator.CompleteRecovery(msgInfo.BatchID);
recoveryTrackingMap.Remove(msgInfo.BatchID);
}
}
}
return(Task.CompletedTask);
}
public Task MarkOperatorAsFailed()
{
isOperatorFailed = true;
isARestartOperator = true;
PrettyConsole.Line("Mark this as failed!");
return(Task.CompletedTask);
}
public Task RemoveCustomDownStreamOperator(Guid guid)
{
int index = -1;
for (int i = 0; i < downStreamOperators.Count; i++)
{
if (downStreamOperators[i].GetPrimaryKey() == guid)
{
index = i;
break;
}
}
if (index != -1)
{
downStreamOperators.RemoveAt(index);
PrettyConsole.Line("Remove old stateful from upper stream");
operatorSettings.RemoveOperatorFromDict(guid);
topologyManager.UpdateOperatorSettings(this.GetPrimaryKey(), operatorSettings);
}
else
{
throw new ArgumentException();
}
return(Task.CompletedTask);
}
private bool CheckCount(StreamMessage msg)
{
if (!upStreamMessageCountMaps.ContainsKey(msg.BatchID))
{
if (msg.Count == 0)
{
return(true);
}
else
{
return(false);
}
}
else if (upStreamMessageCountMaps[msg.BatchID].ContainsKey(msg.From) && msg.Count == upStreamMessageCountMaps[msg.BatchID][msg.From])
{
return(true);
}
else if (!upStreamMessageCountMaps[msg.BatchID].ContainsKey(msg.From) && msg.Count == 0)
{
return(true);
}
else
{
PrettyConsole.Line("The count in stateless operator is not equal!");
return(false);
}
}
//Commit
public Task StartCommit(int ID)
{
PrettyConsole.Line("Start Commit Batch " + ID);
commitMsg.BatchID = ID;
topologyManager.Commit(commitMsg);
return(Task.CompletedTask);
}
public override Task OnNextAsync(ChatMsg item, StreamSequenceToken token = null)
{
if (item.Author == "System")
{
PrettyConsole.Line($"==== System message: '{item.Text}'", ConsoleColor.Green);
}
return(Task.CompletedTask);
}
private async Task <Task> DetectPossibleFailures(object org)
{
PrettyConsole.Line("Start Detect Failures");
Dictionary <Guid, Task <int> > taskMap = new Dictionary <Guid, Task <int> >();
List <Task <int> > taskList = new List <Task <int> >();
var topologyManager = GrainFactory.GetGrain <ITopology>(Constants.Topology_Manager);
var operatorUnits = await topologyManager.GetAllUnits();
foreach (TopologyUnit unit in operatorUnits)
{
if (unit.OperatorType == OperatorType.Source)
{
IStreamSource source = GrainFactory.GetGrain <IStreamSource>(unit.PrimaryKey);
var task = source.DetectErrors();
taskMap.Add(unit.PrimaryKey, task);
}
else if (unit.OperatorType == OperatorType.Stateful)
{
IStatefulOperator statefulOperator = GrainFactory.GetGrain <IStatefulOperator>(unit.PrimaryKey, Constants.Stateful_Operator_Prefix);
var task = statefulOperator.DetectErrors();
taskMap.Add(unit.PrimaryKey, task);
}
else if (unit.OperatorType == OperatorType.Stateless)
{
IStatelessOperator statelessOperator = GrainFactory.GetGrain <IStatelessOperator>(unit.PrimaryKey, Constants.Stateless_Operator_Prefix);
var task = statelessOperator.DetectErrors();
taskMap.Add(unit.PrimaryKey, task);
}
else
{
throw new ArgumentException("Commit: The operator type is in valid!");
}
}
try
{
await Task.WhenAny(Task.WhenAll(taskMap.Values), Task.Delay(TimeSpan.FromSeconds(2)));
}
catch (Exception e)
{
PrettyConsole.Line(e.ToString());
}
foreach (var task in taskMap)
{
if (task.Value.Status != TaskStatus.RanToCompletion)
{
PrettyConsole.Line("Replace!");
await topologyManager.ReplaceTheOldOperator(task.Key);
disposable.Dispose();
break;
}
}
return(Task.CompletedTask);
}
private Task TellTrackMessageSent(StreamMessage item)
{
if (tracker != null)
{
tracker.CompleteTracking(item);
PrettyConsole.Line("Complete one barrier");
}
return(Task.CompletedTask);
}
//Replay Logic
public async Task <Task> ReplayTheMessageOnRecoveryCompleted()
{
PrettyConsole.Line("Start Replay!");
foreach (StreamMessage msg in messagesForRecovery)
{
await ProcessNormalMessage(msg);
}
return(Task.CompletedTask);
}
protected async Task <Task> ExecuteMessagesByDownStreamOperators(StreamMessage msg, IAsyncStream <StreamMessage> stream, IOperator op)
{
if (downStreamOperators.Count > 0)
{
int batchID = msg.BatchID;
var targetKey = op.GetPrimaryKey();
try
{
msg.From = this.GetPrimaryKey();
//if is barrier message, set the message count
if (msg.Value == Constants.Barrier_Value)
{
if (downStreamMessageCountMaps.ContainsKey(batchID))
{
if (downStreamMessageCountMaps[batchID].ContainsKey(op.GetPrimaryKey()))
{
msg.Count = downStreamMessageCountMaps[batchID][op.GetPrimaryKey()];
}
else
{
msg.Count = 0;
}
}
else
{
msg.Count = 0;
}
}
//if it is a normal message, increment the count map
else if (msg.Value != Constants.System_Key)
{
if (!downStreamMessageCountMaps.ContainsKey(batchID))
{
downStreamMessageCountMaps.Add(batchID, new Dictionary <Guid, int>());
}
var key = op.GetPrimaryKey();
if (downStreamMessageCountMaps[batchID].ContainsKey(key))
{
downStreamMessageCountMaps[batchID][key] = downStreamMessageCountMaps[batchID][key] + 1;
}
else
{
downStreamMessageCountMaps[batchID].Add(key, 1);
}
}
op.ExecuteMessage(msg, stream);
}
catch (Exception e)
{
PrettyConsole.Line("Get Exception : " + e + "; Start Receovry");
topologyManager.ReplaceTheOldOperator(targetKey);
}
}
return(Task.CompletedTask);
}
public override Task OnNextAsync(ChatMsg item, StreamSequenceToken token = null)
{
var textHasMention = item.Text.Contains("@");
if (textHasMention)
{
PrettyConsole.Line($"==== MENTION DETECTED: '{item.Author}' mentions someone. Sending a notification email.", ConsoleColor.Green);
}
return(Task.CompletedTask);
}
public override Task OnActivateAsync()
{
currentBatchID = 0;
committedID = -1;
tracker = GrainFactory.GetGrain <IBatchTracker>(Utils.Constants.Tracker);
topologyManager = GrainFactory.GetGrain <ITopology>(Constants.Topology_Manager);
PrettyConsole.Line("Register Timer");
var streamProvider = GetStreamProvider(Constants.FaultTolerantStreamProvider);
return(base.OnActivateAsync());
}
private static void PrintHints()
{
var menuColor = ConsoleColor.Magenta;
PrettyConsole.Line("Type '/j <channel>' to join specific channel", menuColor);
PrettyConsole.Line("Type '/n <username>' to set your user name", menuColor);
PrettyConsole.Line("Type '/l' to leave specific channel", menuColor);
PrettyConsole.Line("Type '<any text>' to send a message", menuColor);
PrettyConsole.Line("Type '/h' to re-read channel history", menuColor);
PrettyConsole.Line("Type '/m' to query members in the channel", menuColor);
PrettyConsole.Line("Type '/exit' to exit client.", menuColor);
}
private static async Task ShowChannelMembers(IClusterClient client)
{
var room = client.GetGrain <IStreamSource>(joinedChannel);
var members = await room.GetMembers();
PrettyConsole.Line($"====== Members for '{joinedChannel}' Channel ======", ConsoleColor.DarkGreen);
foreach (var member in members)
{
PrettyConsole.Line(member, ConsoleColor.DarkGreen);
}
PrettyConsole.Line("============", ConsoleColor.DarkGreen);
}
public static void Main(string[] args)
{
var clientInstance = InitializeClient(args).GetAwaiter().GetResult();
PrettyConsole.Line("==== CLIENT: Initialized ====", ConsoleColor.Cyan);
PrettyConsole.Line("CLIENT: Write commands:", ConsoleColor.Cyan);
Menu(clientInstance).GetAwaiter().GetResult();
PrettyConsole.Line("==== CLIENT: Shutting down ====", ConsoleColor.DarkRed);
}
private static async Task ShowCurrentChannelHistory(IClusterClient client)
{
var room = client.GetGrain <IStreamSource>(joinedChannel);
var history = await room.ReadHistory(1000);
PrettyConsole.Line($"====== History for '{joinedChannel}' Channel ======", ConsoleColor.DarkGreen);
foreach (var chatMsg in history)
{
PrettyConsole.Line($" ({chatMsg.Created:g}) {chatMsg.Key}> {chatMsg.Value}", ConsoleColor.DarkGreen);
}
PrettyConsole.Line("============", ConsoleColor.DarkGreen);
}
//This function get the words and count
public async Task <Task> ExecuteMessage(StreamMessage msg, IAsyncStream <StreamMessage> stream)
{
//At frist, if it is a new batch, just creat the incremental log
//for it
if (!incrementalLogMap.ContainsKey(msg.BatchID))
{
var newIncrementalLog = new Dictionary <string, object>();
var newReverseLog = new Dictionary <string, object>();
incrementalLogMap.Add(msg.BatchID, newIncrementalLog);
reverseLogMap.Add(msg.BatchID, newReverseLog);
}
if (msg.BatchID > currentReverseLogID)
{
currentReverseLogID = msg.BatchID;
}
if (msg.BatchID > currentBatchID)
{
messageBuffer.Add(msg);
asyncStream = stream;
}
else if (msg.BatchID == currentBatchID || msg.Key == Constants.System_Key)
{
if (msg.Key != Constants.System_Key)
{
await IncrementUpStreamCount(msg);
await CustomExecutionMethod(msg, stream);
currentWordsProcessed++;
if (!isARestartOperator && currentWordsProcessed >= numOfMaxProcessWords && isFailureTestOpen)
{
Thread.Sleep(10000);
}
}
else
{
await ProcessSpecialMessage(msg);
if (downStreamOperators.Count > 0)
{
await BroadcastSpecialMessage(msg, stream);
}
}
}
else
{
PrettyConsole.Line("ERRROOOOOOROOOR");
throw new InvalidOperationException(msg.Key + " " + msg.Value + " The id " + msg.BatchID + " is less than the currentID:" + currentBatchID);
}
return(Task.CompletedTask);
}
public Task <bool> Tick(int exceptionMod)
{
counter++;
PrettyConsole.Line($"Tick: {counter}");
if (counter >= 0 && counter % exceptionMod == 0)
{
PrettyConsole.Line("EXCEPTION !!!!", ConsoleColor.Red);
throw new Exception("Bang !!!");
}
return(Task.FromResult(true));
}
public async Task <object> Invoke(MethodInfo method, InvokeMethodRequest request, IGrainMethodInvoker invoker)
{
try
{
return(await invoker.Invoke(this, request));
}
catch (Exception e)
{
PrettyConsole.Line("Exception in interceptor");
DeactivateOnIdle();
throw;
}
}
public Task CompleteCommit(int batchID)
{
if (batchID - committedID == 1)
{
committedID++;
PrettyConsole.Line("Committed Batch ID now is: " + committedID);
return(Task.CompletedTask);
}
else
{
throw new InvalidOperationException("Cannot commit batch greater than committed id 2");
}
}
protected Task SaveStateToFile(IncrementalLog state)
{
PrettyConsole.Line("Save the incremental log to " + operatorSettings.incrementalLogAddress);
try
{
WriteToBinaryFile(operatorSettings.incrementalLogAddress, state);
}
catch (Exception e)
{
PrettyConsole.Line("Error " + e);
}
return(Task.CompletedTask);
}
public Task AddProcessingTime(int time)
{
processingTimeList.Add(time);
if (processingTimeList.Count >= Processing_Time_Interval)
{
numOfWordsProcessed += Processing_Time_Interval;
if (numOfWordsProcessed % 1000 == 0)
{
PrettyConsole.Line("Process " + numOfWordsProcessed + " words");
}
SaveProcessingTimeIntoFiles();
processingTimeList.Clear();
}
return(Task.CompletedTask);
}
private static async Task <IClusterClient> InitializeClient(string[] args)
{
int initializeCounter = 0;
var initSucceed = false;
while (!initSucceed)
{
try
{
var client = new ClientBuilder().Configure <ClusterOptions>(options =>
{
options.ClusterId = Constants.ClusterId;
options.ServiceId = Constants.ServiceId;
})
.UseLocalhostClustering()
.ConfigureApplicationParts(parts => parts.AddApplicationPart(typeof(IStreamSource).Assembly).WithReferences())
.ConfigureLogging(logging => logging.AddConsole())
//Depends on your application requirements, you can configure your client with other stream providers, which can provide other features,
//such as persistence or recoverability. For more information, please see http://dotnet.github.io/orleans/Documentation/Orleans-Streams/Stream-Providers.html
.AddSimpleMessageStreamProvider(Constants.ChatRoomStreamProvider)
.Build();
await client.Connect();
initSucceed = client.IsInitialized;
if (initSucceed)
{
return(client);
}
}
catch (Exception exc)
{
PrettyConsole.Line(exc.Message, ConsoleColor.Cyan);
initSucceed = false;
}
if (initializeCounter++ > 10)
{
return(null);
}
PrettyConsole.Line("Client Init Failed. Sleeping 5s...", ConsoleColor.Red);
Thread.Sleep(TimeSpan.FromSeconds(5));
}
return(null);
}
//If it is special message, it has to send to all the operators.
//If it is barrier message, batch manager will start to track it
//by using BarrierMsgTrackingInfo which keep and ID and the number of
//client it sent to.
private async Task <Task> ProcessSpecialMessage(StreamMessage msg, IAsyncStream <StreamMessage> stream)
{
BarrierOrCommitMsgTrackingInfo info = new BarrierOrCommitMsgTrackingInfo(msg.barrierOrCommitInfo.GetID(), msg.barrierOrCommitInfo.numberOfClientSent);
info.BatchID = msg.BatchID;
if (msg.Value == Constants.Barrier_Value && msg.BatchID - currentBatchID < 2)
{
currentBatchID = msg.BatchID + 1;
PrettyConsole.Line("Increment ID " + currentBatchID);
await TrackingBarrierMessages(msg);
await BroadcastSpecialMessage(msg, stream);
await batchTracker.CompleteOneOperatorBarrier(info);
}
return(Task.CompletedTask);
}
public async Task <Task> SaveIncrementalLogIntoStorage()
{
//Once save the state to files, then clear
//The incremental log
if ((currentBatchID % Constants.Checkpoint_Interval == 0) && currentBatchID != 0)
{
await SaveStateToFile(new IncrementalLog(statesMap, currentBatchID, LogType.CheckPoint));
PrettyConsole.Line("Save checkpoint: " + currentBatchID % Constants.Checkpoint_Interval + " in batch: " + currentBatchID);
}
else
{
var incrementalLog = await GetIncrementalLog(currentBatchID);
await SaveStateToFile(new IncrementalLog(incrementalLog, currentBatchID, LogType.Incremental));
}
return(Task.CompletedTask);
}
//Once the recovery completed, just restart the timer
//Restart the timer
public async Task <Task> CompleteRecovery(int batchID)
{
if (committedID == batchID)
{
currentBatchID = batchID + 1;
disposable = RegisterTimer(SendBarrierOnPeriodOfTime, null, barrierTimeInterval, barrierTimeInterval);
var detector = GrainFactory.GetGrain <IErrorDetector>(Constants.Error_Detector);
detector.RegisterTimerToDetectFailures();
var recoveryTime = System.DateTime.Now.Millisecond - startTime;
PrettyConsole.Line("Recovery Time: " + recoveryTime);
return(Task.CompletedTask);
}
else
{
throw new InvalidOperationException("The recvoery batch is not equal to the latest committed ID");
}
}
public async Task <Task> AddCustomeOperatorsToNonSourceOperators(List <IOperator> upperOps, List <IOperator> downOps)
{
List <TopologyUnit> units = new List <TopologyUnit>();
foreach (var op in downOps)
{
units.Add(await op.GetTopologyUnit());
}
foreach (var op in upperOps)
{
op.AddCustomDownStreamOperators(units);
PrettyConsole.Line("Add 1");
}
return(Task.CompletedTask);
}
private static async Task LeaveChannel(IClusterClient client)
{
PrettyConsole.Line($"Leaving channel {joinedChannel}");
var room = client.GetGrain <IStreamSource>(joinedChannel);
var streamId = await room.Leave(userName);
var stream = client.GetStreamProvider(Constants.ChatRoomStreamProvider)
.GetStream <StreamMessage>(streamId, Constants.CharRoomStreamNameSpace);
//unsubscribe from the channel/stream since client left, so that client won't
//receive furture messages from this channel/stream
var subscriptionHandles = await stream.GetAllSubscriptionHandles();
foreach (var handle in subscriptionHandles)
{
await handle.UnsubscribeAsync();
}
}
