public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
ulong filesize = ((ScanPredicate)predicate).FileSize;
tableId = ((ScanPredicate)predicate).TableID;
separator = ((ScanPredicate)predicate).Separator;
string extensionKey = "";
Guid key = this.GetPrimaryKey(out extensionKey);
ulong i = UInt64.Parse(extensionKey);
ulong num_grains = (ulong)((ScanPredicate)predicate).NumberOfGrains;
ulong partition = filesize / num_grains;
ulong start_byte = i * partition;
ulong end_byte = num_grains - 1 == i?filesize:(i + 1) * partition;
reader = new ScanStreamReader(((ScanPredicate)predicate).File);
if (!reader.GetFile(start_byte))
{
return(Task.FromException(new Exception("unable to get file")));
}
start = start_byte;
end = end_byte;
if (start != 0)
{
start += reader.TrySkipFirst();
}
Console.WriteLine("Init: start byte: " + start.ToString() + " end byte: " + end.ToString());
return(Task.CompletedTask);
}
public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
searchIndex = ((KeywordPredicate)predicate).SearchIndex;
keyword = ((KeywordPredicate)predicate).Query;
return(Task.CompletedTask);
}
public override async Task Build(IPrincipalGrain principal, IGrainFactory gf, List <Pair <Operator, WorkerLayer> > prev)
{
layer = new Dictionary <SiloAddress, List <IWorkerGrain> >();
for (int i = 0; i < numWorkers; ++i)
{
IWorkerGrain grain = gf.GetGrain <IWorkerGrain>(principal.GetPrimaryKey(), id + i.ToString());
RequestContext.Clear();
if (deployArgs != null)
{
foreach (var p in deployArgs)
{
RequestContext.Set(p.Key, p.Value);
}
}
RequestContext.Set("excludeSilo", Constants.ClientIPAddress);
RequestContext.Set("grainIndex", i);
SiloAddress addr = await grain.Init(principal, workerGen(i));
Console.WriteLine("Placement: " + OperatorImplementation.Common.Utils.GetReadableName(grain) + " placed at " + addr);
if (!layer.ContainsKey(addr))
{
layer.Add(addr, new List <IWorkerGrain> {
grain
});
}
else
{
layer[addr].Add(grain);
}
}
}
public override async Task BuildWorkerTopology()
{
//build backward
//2-layer
operatorGrains = Enumerable.Range(0, 2).Select(x => new List <IWorkerGrain>()).ToList();
//last layer
IWorkerGrain finalGrain = this.GrainFactory.GetGrain <ICountFinalOperatorGrain>(this.GetPrimaryKey(), "final");
operatorGrains[1].Add(finalGrain);
//first layer
ISendStrategy strategy = new RoundRobin(new List <IWorkerGrain> {
finalGrain
});
for (int i = 0; i < DefaultNumGrainsInOneLayer; ++i)
{
IWorkerGrain grain = this.GrainFactory.GetGrain <ICountOperatorGrain>(this.GetPrimaryKey(), i.ToString());
await grain.SetSendStrategy(this.GetPrimaryKey(), strategy);
operatorGrains[0].Add(grain);
}
//set target end flag
await finalGrain.SetInputInformation(new Dictionary <string, int> {
{ this.GetPrimaryKeyString(), DefaultNumGrainsInOneLayer }
});
}
public virtual Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
this.self = self;
this.principalGrain = principalGrain;
this.predicate = predicate;
return(Task.CompletedTask);
}
public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
joinFieldIndex = ((HashJoinPredicate)predicate).JoinFieldIndex;
TableID = ((HashJoinPredicate)predicate).TableID;
return(Task.CompletedTask);
}
public string ReturnGrainIndentifierString(IWorkerGrain grain)
{
//string a="Engine.OperatorImplementation.Operators.OrleansCodeGen";
string extension;
//grain.GetPrimaryKey(out extension);
return(grain.GetPrimaryKey(out extension).ToString() + " " + extension);
}
public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
groupByIndex = ((GroupByPredicate)predicate).GroupByIndex;
aggregationFunc = ((GroupByPredicate)predicate).AggregationFunction;
aggregationIndex = ((GroupByPredicate)predicate).AggregationIndex;
return(Task.CompletedTask);
}
public Task OnWorkerReceivedAllBatches(IWorkerGrain sender)
{
flags++;
if (flags == workerStates.Count)
{
controllerGrain.OnPrincipalReceivedAllBatches(self);
}
return(Task.CompletedTask);
}
public Task <SiloAddress> Init(IPrincipalGrain principalGrain, ITupleProducer producer)
{
this.self = this.GrainReference.Cast <IWorkerGrain>();
Console.WriteLine("Init Start: " + Utils.GetReadableName(self));
this.principalGrain = principalGrain;
this.producer = producer;
Console.WriteLine("Init Finished: " + Utils.GetReadableName(self));
return(Task.FromResult(localSiloDetails.SiloAddress));
}
protected async Task SendMessageTo(IWorkerGrain nextGrain, Immutable <PayloadMessage> message, int retryCount)
{
nextGrain.ReceivePayloadMessage(message).ContinueWith((t) =>
{
if (Utils.IsTaskTimedOutAndStillNeedRetry(t, retryCount))
{
SendMessageTo(nextGrain, message, retryCount + 1);
}
});
}
public bool Accept(IWorkerGrain sender, LocalBreakpointBase breakpoint)
{
if (breakpoint.version == version && unReportedWorkers.Contains(sender))
{
unReportedWorkers.Remove(sender);
AcceptImpl(sender, breakpoint);
return(true);
}
return(false);
}
public Task OnWorkerFinished(IWorkerGrain sender)
{
workerStates[sender] = WorkerState.Completed;
if (workerStates.Values.All(x => x == WorkerState.Completed))
{
Console.WriteLine("Principal: " + this.GetPrimaryKey() + " completed!");
controllerGrain.OnPrincipalCompleted(self);
}
return(Task.CompletedTask);
}
public virtual async Task BuildWorkerTopology()
{
operatorGrains = Enumerable.Range(0, 1).Select(x => new List <IWorkerGrain>()).ToList();
//one-layer init
for (int i = 0; i < DefaultNumGrainsInOneLayer; ++i)
{
IWorkerGrain grain = GetOperatorGrain(i.ToString());
operatorGrains[0].Add(grain);
}
// for multiple-layer init, do some linking inside...
}
public void AddReceiver(IWorkerGrain receiver, bool localSending = false)
{
if (localSending)
{
unit = new SendingUnit(receiver);
}
else
{
unit = new FlowControlUnit(receiver);
}
}
public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
if (typeof(T) != typeof(string) && (ParseInfo == null || !typeof(T).IsAssignableFrom(ParseInfo.ReturnType)))
{
throw new InvalidOperationException("Invalid type, must contain public static T Parse(string)");
}
type = ((FilterPredicate)predicate).Type;
filterIndex = ((FilterPredicate)predicate).FilterIndex;
threshold = Parse(((FilterPredicate)predicate).Threshold);
return(Task.CompletedTask);
}
public void AddReceiver(IWorkerGrain receiver, bool localSending)
{
if (localSending)
{
localSender += 1;
receivers.Add(new SendingUnit(receiver));
}
else
{
receivers.Add(new FlowControlUnit(receiver));
}
this.outputSequenceNumbers.Add(0);
}
public Task OnWorkerLocalBreakpointTriggered(IWorkerGrain sender, List <LocalBreakpointBase> breakpoint)
{
if (!stateTransitioning)
{
needResume = true;
self.Pause();
}
foreach (LocalBreakpointBase bp in breakpoint)
{
triggeredBreakpointIDs.Add(bp.id);
savedBreakpoints[bp.id].Accept(sender, bp);
}
return(Task.CompletedTask);
}
public override void AddReceiver(IWorkerGrain receiver, bool localSending)
{
if (localSending)
{
localSender += 1;
receivers.Add(new SendingUnit(receiver));
}
else
{
receivers.Add(new FlowControlUnit(receiver));
}
this.outputSequenceNumbers.Add(0);
this.outputRows.Add(new Queue <TexeraTuple>());
}
public virtual async Task <SiloAddress> Init(IPrincipalGrain principalGrain, ITupleProcessor processor)
{
this.self = this.GrainReference.Cast <IWorkerGrain>();
Console.WriteLine("Init Start: " + Utils.GetReadableName(self));
string ext;
this.GetPrimaryKey(out ext);
this.orderingEnforcer = new OrderingGrainWithSequenceNumber(ext);
this.principalGrain = principalGrain;
this.processor = processor;
await this.processor.Initialize();
Console.WriteLine("Init Finished: " + Utils.GetReadableName(self));
return(localSiloDetails.SiloAddress);
}
public Task AddInputInformation(IWorkerGrain sender)
{
string ext;
Guid id = sender.GetPrimaryKey(out ext);
if (unFinishedUpstream.ContainsKey(id))
{
unFinishedUpstream[id].Add(sender);
}
else
{
unFinishedUpstream[id] = new HashSet <IGrain> {
sender
};
}
targetflags++;
return(Task.CompletedTask);
}
public async Task OnWorkerDidPaused(IWorkerGrain sender)
{
if (workerStates[sender] != WorkerState.Completed)
{
workerStates[sender] = WorkerState.Paused;
if (workerStates.Values.Where(x => x != WorkerState.Completed).All(x => x == WorkerState.Paused))
{
Console.WriteLine("Principal: " + Utils.GetReadableName(self) + " paused");
stateTransitioning = false;
controllerGrain.OnPrincipalPaused(self);
//query breakpoints
foreach (var id in triggeredBreakpointIDs)
{
var bp = savedBreakpoints[id];
await bp.Collect();
if (bp.isTriggered)
{
await controllerGrain.OnBreakpointTriggered(bp.Report());
needResume = false;
}
if (bp.IsRepartitionRequired)
{
operatorCore.AssignBreakpoint(grainLayers, workerStates, savedBreakpoints[id]);
}
else if (bp.IsCompleted)
{
await bp.Remove();
}
}
if (needResume)
{
needResume = false;
await Resume();
}
ProcessNextControlMessage();
}
}
else
{
Console.WriteLine("Warning: " + Utils.GetReadableName(sender) + " tries to become Paused after Completed");
}
}
public void AddReceiver(IWorkerGrain receiver)
{
throw new NotImplementedException();
}
protected override void AcceptImpl(IWorkerGrain sender, LocalBreakpointBase breakpoint)
{
current += ((LocalCountBreakpoint)breakpoint).current;
}
public FlowControlUnit(IWorkerGrain receiver) : base(receiver)
{
}
public override void AddReceiver(IWorkerGrain receiver)
{
receivers.Add(receiver);
this.outputSequenceNumbers.Add(0);
this.outputRows.Add(new Queue <TexeraTuple>());
}
public abstract void AddReceiver(IWorkerGrain receiver);
public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
sortIndex = ((SortPredicate)predicate).SortIndex;
return(Task.CompletedTask);
}
protected abstract void AcceptImpl(IWorkerGrain sender, LocalBreakpointBase breakpoint);
public void AddReceiver(IWorkerGrain receiver)
{
receivers.Add(receiver);
this.outputSequenceNumbers.Add(0);
}