public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
searchIndex = ((KeywordPredicate)predicate).SearchIndex;
keyword = ((KeywordPredicate)predicate).Query;
return(Task.CompletedTask);
}
public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
joinFieldIndex = ((HashJoinPredicate)predicate).JoinFieldIndex;
TableID = ((HashJoinPredicate)predicate).TableID;
return(Task.CompletedTask);
}
public virtual Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
this.self = self;
this.principalGrain = principalGrain;
this.predicate = predicate;
return(Task.CompletedTask);
}
public override async Task Build(IPrincipalGrain principal, IGrainFactory gf, List <Pair <Operator, WorkerLayer> > prev)
{
layer = new Dictionary <SiloAddress, List <IWorkerGrain> >();
for (int i = 0; i < numWorkers; ++i)
{
IWorkerGrain grain = gf.GetGrain <IWorkerGrain>(principal.GetPrimaryKey(), id + i.ToString());
RequestContext.Clear();
if (deployArgs != null)
{
foreach (var p in deployArgs)
{
RequestContext.Set(p.Key, p.Value);
}
}
RequestContext.Set("excludeSilo", Constants.ClientIPAddress);
RequestContext.Set("grainIndex", i);
SiloAddress addr = await grain.Init(principal, workerGen(i));
Console.WriteLine("Placement: " + OperatorImplementation.Common.Utils.GetReadableName(grain) + " placed at " + addr);
if (!layer.ContainsKey(addr))
{
layer.Add(addr, new List <IWorkerGrain> {
grain
});
}
else
{
layer[addr].Add(grain);
}
}
}
public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
ulong filesize = ((ScanPredicate)predicate).FileSize;
tableId = ((ScanPredicate)predicate).TableID;
separator = ((ScanPredicate)predicate).Separator;
string extensionKey = "";
Guid key = this.GetPrimaryKey(out extensionKey);
ulong i = UInt64.Parse(extensionKey);
ulong num_grains = (ulong)((ScanPredicate)predicate).NumberOfGrains;
ulong partition = filesize / num_grains;
ulong start_byte = i * partition;
ulong end_byte = num_grains - 1 == i?filesize:(i + 1) * partition;
reader = new ScanStreamReader(((ScanPredicate)predicate).File);
if (!reader.GetFile(start_byte))
{
return(Task.FromException(new Exception("unable to get file")));
}
start = start_byte;
end = end_byte;
if (start != 0)
{
start += reader.TrySkipFirst();
}
Console.WriteLine("Init: start byte: " + start.ToString() + " end byte: " + end.ToString());
return(Task.CompletedTask);
}
public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
groupByIndex = ((GroupByPredicate)predicate).GroupByIndex;
aggregationFunc = ((GroupByPredicate)predicate).AggregationFunction;
aggregationIndex = ((GroupByPredicate)predicate).AggregationIndex;
return(Task.CompletedTask);
}
public Task <SiloAddress> Init(IPrincipalGrain principalGrain, ITupleProducer producer)
{
this.self = this.GrainReference.Cast <IWorkerGrain>();
Console.WriteLine("Init Start: " + Utils.GetReadableName(self));
this.principalGrain = principalGrain;
this.producer = producer;
Console.WriteLine("Init Finished: " + Utils.GetReadableName(self));
return(Task.FromResult(localSiloDetails.SiloAddress));
}
private async Task SendResumeToNextPrincipalGrain(IPrincipalGrain nextGrain, int retryCount)
{
nextGrain.Resume().ContinueWith((t) =>
{
if (Utils.IsTaskTimedOutAndStillNeedRetry(t, retryCount))
{
SendResumeToNextPrincipalGrain(nextGrain, retryCount + 1);
}
});
}
public Task OnPrincipalPaused(IPrincipalGrain sender)
{
isPrincipalPaused[sender] = true;
if (isPrincipalPaused.Values.All(x => x == true))
{
TimeSpan duration = DateTime.UtcNow - actionStart;
Console.WriteLine("Workflow Paused in " + duration);
performingAction = false;
}
return(Task.CompletedTask);
}
public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
if (typeof(T) != typeof(string) && (ParseInfo == null || !typeof(T).IsAssignableFrom(ParseInfo.ReturnType)))
{
throw new InvalidOperationException("Invalid type, must contain public static T Parse(string)");
}
type = ((FilterPredicate)predicate).Type;
filterIndex = ((FilterPredicate)predicate).FilterIndex;
threshold = Parse(((FilterPredicate)predicate).Threshold);
return(Task.CompletedTask);
}
public Task OnPrincipalReceivedAllBatches(IPrincipalGrain sender)
{
if (nodesKeepedTuples.Contains(sender))
{
timer.Stop();
Console.WriteLine(OperatorImplementation.Common.Utils.GetReadableName(sender) + " received all inputs");
Console.WriteLine("Stage(" + String.Join(',', stageContains) + ") took " + timer.Elapsed);
stageContains.Clear();
sender.Resume();
timer.Restart();
}
return(Task.CompletedTask);
}
public virtual async Task <SiloAddress> Init(IPrincipalGrain principalGrain, ITupleProcessor processor)
{
this.self = this.GrainReference.Cast <IWorkerGrain>();
Console.WriteLine("Init Start: " + Utils.GetReadableName(self));
string ext;
this.GetPrimaryKey(out ext);
this.orderingEnforcer = new OrderingGrainWithSequenceNumber(ext);
this.principalGrain = principalGrain;
this.processor = processor;
await this.processor.Initialize();
Console.WriteLine("Init Finished: " + Utils.GetReadableName(self));
return(localSiloDetails.SiloAddress);
}
public async Task Init(IControllerGrain controllerGrain, Guid workflowID, Operator currentOperator)
{
this.controllerGrain = controllerGrain;
this.workflowID = workflowID;
this.operatorID = currentOperator.OperatorGuid;
this.self = currentOperator.PrincipalGrain;
this.predicate = currentOperator.Predicate;
await BuildWorkerTopology();
PassExtraParametersByPredicate(ref this.predicate);
foreach (List <IWorkerGrain> grainList in operatorGrains)
{
foreach (IWorkerGrain grain in grainList)
{
await grain.Init(grain, predicate, self);
}
}
}
public async Task OnPrincipalCompleted(IPrincipalGrain sender)
{
Console.WriteLine("Controller: " + sender.GetPrimaryKey() + " completed!");
Guid id = sender.GetPrimaryKey();
var itemToDelete = new List <Guid>();
stageContains.Add(nodeMetadata[id].GetType().Name);
if (nodeMetadata[id].GetType() == typeof(ScanOperator))
{
timer.Stop();
Console.WriteLine("Stage(" + String.Join(',', stageContains) + ") took " + timer.Elapsed);
stageContains.Clear();
timer.Restart();
}
if (nodeMetadata[id].GetType().Name.Contains("Sort"))
{
timer.Stop();
Console.WriteLine("Stage(" + String.Join(',', stageContains) + ") took " + timer.Elapsed);
}
foreach (var pair in startDependencies)
{
if (pair.Value.Contains(id))
{
pair.Value.Remove(id);
}
if (pair.Value.Count == 0)
{
timer.Stop();
Console.WriteLine("Stage(" + String.Join(',', stageContains) + ") took " + timer.Elapsed);
stageContains.Clear();
itemToDelete.Add(pair.Key);
await nodes[pair.Key].Start();
timer.Restart();
}
}
foreach (var item in itemToDelete)
{
startDependencies.Remove(item);
}
}
public virtual async Task Init(IControllerGrain controllerGrain, Operator op, List <Pair <Operator, WorkerLayer> > prev)
{
this.controllerGrain = controllerGrain;
this.self = this.GrainReference.Cast <IPrincipalGrain>();
this.operatorCore = op;
var topology = this.operatorCore.GenerateTopology();
grainLayers = topology.First;
links = topology.Second;
foreach (var layer in grainLayers)
{
await layer.Build(self, GrainFactory, prev);
foreach (IWorkerGrain worker in layer.Layer.Values.SelectMany(x => x))
{
workerStates[worker] = WorkerState.UnInitialized;
}
}
foreach (var link in links)
{
await link.Link();
}
}
public Task AddPrevPrincipalGrain(IPrincipalGrain prevGrain)
{
prevPrincipalGrains.Add(prevGrain);
return(Task.CompletedTask);
}
public Task AddNextPrincipalGrain(IPrincipalGrain nextGrain)
{
nextPrincipalGrains.Add(nextGrain);
return(Task.CompletedTask);
}
public virtual void SetPrincipalGrain(IGrainFactory factory)
{
PrincipalGrain = factory.GetGrain <IPrincipalGrain>(OperatorGuid, "Principal");
}
private string ReturnGrainIndentifierString(IPrincipalGrain grain)
{
string extension;
return(grain.GetPrimaryKey(out extension).ToString() + extension);
}
public abstract Task Build(IPrincipalGrain principal, IGrainFactory gf, List <Pair <Operator, WorkerLayer> > prev);
public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
TableID = ((CrossRippleJoinPredicate)predicate).TableID;
return(Task.CompletedTask);
}
private async Task InitOperators(bool checkpointActivated)
{
//topological ordering
var current = nodeMetadata.Keys.Where(x => !backwardLinks.ContainsKey(x)).ToList();
while (current.Count > 0)
{
foreach (var id in current)
{
IPrincipalGrain principal = GrainFactory.GetGrain <IPrincipalGrain>(id);
nodes.Add(id, principal);
RequestContext.Clear();
RequestContext.Set("targetSilo", Constants.ClientIPAddress);
if (backwardLinks.ContainsKey(id))
{
List <Pair <Operator, WorkerLayer> > prev = new List <Pair <Operator, WorkerLayer> >();
List <Guid> prevIDs = new List <Guid>();
foreach (var prevID in backwardLinks[id])
{
if (!checkpointActivated && (nodeMetadata[id].GetType() == typeof(HashJoinOperator) || nodeMetadata[id].GetType() == typeof(GroupByFinalOperator)))
{
nodesStashedTuples.Add(nodes[prevID]);
}
var t = await nodes[prevID].GetOutputLayer();
prevIDs.Add(prevID);
prev.Add(new Pair <Operator, WorkerLayer>(nodeMetadata[prevID], t));
}
await principal.Init(self, nodeMetadata[id], prev);
if (!checkpointActivated && (nodeMetadata[id].GetType() == typeof(HashJoinOperator) || nodeMetadata[id].GetType() == typeof(GroupByFinalOperator)))
{
principal.Pause();
nodesKeepedTuples.Add(principal);
}
var inputLayer = await principal.GetInputLayer();
for (int i = 0; i < prev.Count; ++i)
{
var pair = prev[i];
var prevID = prevIDs[i];
if (nodeMetadata[prevID].GetType() == typeof(HashBasedFolderScanOperator))
{
var link = new OneToOneLinking(pair.Second, inputLayer, Constants.BatchSize);
await link.Link();
nodeLinks.Add(link);
}
else if (nodeMetadata[id].GetType().Name.Contains("Sort"))
{
var link = new AllToOneLinking(pair.Second, inputLayer, Constants.BatchSize);
await link.Link();
nodeLinks.Add(link);
}
else if (nodeMetadata[id].IsStaged(pair.First))
{
var link = new OneToOneLinking(pair.Second, inputLayer, Constants.BatchSize);
await link.Link();
nodeLinks.Add(link);
}
else
{
var link = new HashBasedShuffleLinking(nodeMetadata[id].GetHashFunctionAsString(prevID), pair.Second, inputLayer, Constants.BatchSize);
await link.Link();
nodeLinks.Add(link);
}
}
}
else
{
await principal.Init(self, nodeMetadata[id], null);
}
}
current = current.SelectMany(x =>
{
if (forwardLinks.ContainsKey(x))
{
return(forwardLinks[x]);
}
else
{
return(Enumerable.Empty <Guid>());
}
}).Where(x => backwardLinks[x].All(y => nodes.ContainsKey(y))).Distinct().ToList();
}
}
public Task OnPrincipalRunning(IPrincipalGrain sender)
{
return(Task.CompletedTask);
}
public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
sortIndex = ((SortPredicate)predicate).SortIndex;
return(Task.CompletedTask);
}
public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain)
{
base.Init(self, predicate, principalGrain);
projectionIndexs = ((ProjectionPredicate)predicate).ProjectionIndexs;
return(Task.CompletedTask);
}
