public async Task <Task> Recovery(StreamMessage msg)
{
List <TopologyUnit> units = topology.GetAllTopologyUnits();
msg.barrierOrCommitInfo = new BarrierOrCommitMsgTrackingInfo(Guid.NewGuid(), units.Count);
msg.barrierOrCommitInfo.BatchID = msg.BatchID;
await batchTracker.TrackingRecoveryMessages(msg);
foreach (TopologyUnit unit in units)
{
if (unit.OperatorType == OperatorType.Source)
{
IStreamSource source = GrainFactory.GetGrain <IStreamSource>(unit.PrimaryKey);
source.Recovery(msg);
}
else if (unit.OperatorType == OperatorType.Stateful)
{
IStatefulOperator statefulOperator = GrainFactory.GetGrain <IStatefulOperator>(unit.PrimaryKey, Constants.Stateful_Operator_Prefix);
statefulOperator.Recovery(msg);
}
else if (unit.OperatorType == OperatorType.Stateless)
{
IStatelessOperator statelessOperator = GrainFactory.GetGrain <IStatelessOperator>(unit.PrimaryKey, Constants.Stateless_Operator_Prefix);
statelessOperator.Recovery(msg);
}
else
{
throw new ArgumentException("Recovery: The operator type is in valid!");
}
}
return(Task.CompletedTask);
}
public OperatorStateReader(IOperatorStateReaderFactory factory, ISerializer serializer, Stream stream, IStatefulOperator @operator)
{
Debug.Assert(factory != null);
Debug.Assert(serializer != null);
Debug.Assert(stream != null);
_factory = factory;
_serializer = serializer;
_operator = @operator;
_underlyingStream = stream;
_originalPosition = stream.Position;
_end = _originalPosition;
var lenBytes = new byte[8];
var readCount = stream.Read(lenBytes, 0, 8);
if (readCount != 8 && @operator is not ITransitioningOperator)
{
var blob = stream.GetBase64Blob();
throw new InvalidDataException(string.Format(CultureInfo.InvariantCulture, "Missing length prefix. Operator = {0}/{1}, Position = {2}, State = {3}", _operator.Name, _operator.Version, _originalPosition, blob));
}
else if (readCount == 8)
{
var length = BitConverter.ToInt64(lenBytes, 0);
_stream = new StreamSegment(stream, stream.Position, length);
_end = stream.Position + length;
}
}
private async Task <Task> DetectPossibleFailures(object org)
{
PrettyConsole.Line("Start Detect Failures");
Dictionary <Guid, Task <int> > taskMap = new Dictionary <Guid, Task <int> >();
List <Task <int> > taskList = new List <Task <int> >();
var topologyManager = GrainFactory.GetGrain <ITopology>(Constants.Topology_Manager);
var operatorUnits = await topologyManager.GetAllUnits();
foreach (TopologyUnit unit in operatorUnits)
{
if (unit.OperatorType == OperatorType.Source)
{
IStreamSource source = GrainFactory.GetGrain <IStreamSource>(unit.PrimaryKey);
var task = source.DetectErrors();
taskMap.Add(unit.PrimaryKey, task);
}
else if (unit.OperatorType == OperatorType.Stateful)
{
IStatefulOperator statefulOperator = GrainFactory.GetGrain <IStatefulOperator>(unit.PrimaryKey, Constants.Stateful_Operator_Prefix);
var task = statefulOperator.DetectErrors();
taskMap.Add(unit.PrimaryKey, task);
}
else if (unit.OperatorType == OperatorType.Stateless)
{
IStatelessOperator statelessOperator = GrainFactory.GetGrain <IStatelessOperator>(unit.PrimaryKey, Constants.Stateless_Operator_Prefix);
var task = statelessOperator.DetectErrors();
taskMap.Add(unit.PrimaryKey, task);
}
else
{
throw new ArgumentException("Commit: The operator type is in valid!");
}
}
try
{
await Task.WhenAny(Task.WhenAll(taskMap.Values), Task.Delay(TimeSpan.FromSeconds(2)));
}
catch (Exception e)
{
PrettyConsole.Line(e.ToString());
}
foreach (var task in taskMap)
{
if (task.Value.Status != TaskStatus.RanToCompletion)
{
PrettyConsole.Line("Replace!");
await topologyManager.ReplaceTheOldOperator(task.Key);
disposable.Dispose();
break;
}
}
return(Task.CompletedTask);
}
/// <summary>
/// Loads the state of the specified stateful operator from the specified state reader.
/// </summary>
/// <param name="reader">Reader to read operator state from.</param>
/// <param name="node">Operator whose state to read.</param>
public static void LoadState(this IOperatorStateReader reader, IStatefulOperator node)
{
var version = LoadVersionInfo(reader, node);
if (version != null)
{
node.LoadState(reader, version);
}
else
{
reader.Reset();
}
}
public async Task <Task> InitRandomOperators()
{
IStatefulOperator operatorOne = GrainFactory.GetGrain <IStatefulOperator>(Guid.NewGuid());
IStatefulOperator operatorTwo = GrainFactory.GetGrain <IStatefulOperator>(Guid.NewGuid());
downStreamOperators.Add(operatorOne);
operatorOne.IncrementNumberOfUpStreamOperator();
downStreamOperators.Add(operatorTwo);
operatorTwo.IncrementNumberOfUpStreamOperator();
operatorSettings.AddOpratorToDict(operatorOne.GetPrimaryKey(), await operatorOne.GetOperatorSettings());
operatorSettings.AddOpratorToDict(operatorTwo.GetPrimaryKey(), await operatorTwo.GetOperatorSettings());
topologyManager.UpdateOperatorSettings(this.GetPrimaryKey(), operatorSettings);
topologyManager.ConnectUnits(this.GetPrimaryKey(), operatorOne.GetPrimaryKey());
topologyManager.ConnectUnits(this.GetPrimaryKey(), operatorTwo.GetPrimaryKey());
return(Task.CompletedTask);
}
/// <summary>
/// Saves the state of the specified stateful operator using a writer obtained from the specified state writer factory.
/// </summary>
/// <param name="factory">Factory to create an operator state writer to write operator state to.</param>
/// <param name="node">Operator whose state to write.</param>
public static void SaveState(this IOperatorStateWriterFactory factory, IStatefulOperator node)
{
using var writer = factory.Create(node);
SaveState(writer, node);
}
/// <summary>
/// Saves the state of the specified stateful operator to the specified state writer.
/// </summary>
/// <param name="writer">Writer to write operator state to.</param>
/// <param name="node">Operator whose state to write.</param>
public static void SaveState(this IOperatorStateWriter writer, IStatefulOperator node)
{
SaveVersionInfo(writer, node);
node.SaveState(writer, node.Version);
}
/// <summary>
/// Loads the state of the specified stateful operator using a reader obtained from the specified state reader factory.
/// </summary>
/// <param name="factory">Factory to create an operator state reader to read operator state from.</param>
/// <param name="node">Operator whose state to read.</param>
public static void LoadState(this IOperatorStateReaderFactory factory, IStatefulOperator node)
{
using var reader = factory.Create(node);
LoadState(reader, node);
}
/// <summary>
/// Creates a new operator state reader for the specified operator.
/// </summary>
/// <param name="node">Operator whose state will be read by the created reader.</param>
/// <returns>Operator state reader for the specified operator.</returns>
public IOperatorStateReader Create(IStatefulOperator node)
{
Debug.Assert(_serializer != null, "Did you forget to call ReadHeader?");
return(new OperatorStateReader(this, _serializer, _stream, node));
}
public async Task <Task> ReplaceTheOldOperator(Guid oldGuid)
{
var oldUnit = topology.GetUnit(oldGuid);
var newUnit = new TopologyUnit(oldUnit.OperatorType, Guid.NewGuid());
var newGuid = newUnit.PrimaryKey;
//Only the stateful load the settings
if (newUnit.OperatorType == OperatorType.Stateful)
{
IStatefulOperator statefulOp = GrainFactory.GetGrain <IStatefulOperator>(newUnit.PrimaryKey, Constants.Stateful_Operator_Prefix);
await statefulOp.MarkOperatorAsFailed();
await statefulOp.LoadSettings(oldUnit.GetSettings());
}
//Disconnect the old and connect new
var upperStreamUnits = oldUnit.GetUpperStreamUnits();
var downsStreamUnits = oldUnit.GetdownStreamUnits();
PrettyConsole.Line("Number of upperStream : " + upperStreamUnits.Count);
PrettyConsole.Line("Number of downStream : " + downsStreamUnits.Count);
//Handle Upper Stream
if (upperStreamUnits.Count > 0)
{
var keyList = upperStreamUnits.Keys.ToList();
int index = 0;
foreach (var item in upperStreamUnits.Values.ToList())
{
await DisConnectUnits(item.PrimaryKey, oldGuid);
IOperator op;
var unitList = new List <TopologyUnit>();
unitList.Add(newUnit);
if (item.OperatorType == OperatorType.Stateless)
{
op = GrainFactory.GetGrain <IOperator>(keyList[index], Constants.Stateless_Operator_Prefix);
await op.AddCustomDownStreamOperators(unitList);
await op.RemoveCustomDownStreamOperator(oldGuid);
}
else if (item.OperatorType == OperatorType.Stateful)
{
op = GrainFactory.GetGrain <IOperator>(keyList[index], Constants.Stateful_Operator_Prefix);
await op.AddCustomDownStreamOperators(unitList);
await op.RemoveCustomDownStreamOperator(oldGuid);
}
else if (item.OperatorType == OperatorType.Source)
{
var source = GrainFactory.GetGrain <IStreamSource>(item.PrimaryKey);
await source.AddCustomDownStreamOperators(unitList);
await source.RemoveCustomDownStreamOperator(oldGuid);
}
index++;
}
}
//Handle Down Stream
if (downsStreamUnits.Count > 0)
{
var units = downsStreamUnits.Values.ToList();
IOperator newOp;
if (newUnit.OperatorType == OperatorType.Stateless)
{
newOp = GrainFactory.GetGrain <IStatelessOperator>(newGuid, Constants.Stateless_Operator_Prefix);
await newOp.AddCustomDownStreamOperators(units);
}
else if (newUnit.OperatorType == OperatorType.Stateful)
{
newOp = GrainFactory.GetGrain <IStatefulOperator>(newGuid, Constants.Stateful_Operator_Prefix);
await newOp.AddCustomDownStreamOperators(units);
}
else
{
throw new ArgumentException("Source cannot be down stream operator");
}
}
//Remove the old
await RemoveUnit(oldGuid);
//Start Recovery
var batchCoordinator = GrainFactory.GetGrain <IBatchCoordinator>(Constants.Coordinator);
batchCoordinator.StartRecovery();
return(Task.CompletedTask);
}
public IOperatorStateWriter Create(IStatefulOperator o)
{
return(this);
}
