private ParallelState GenerateParallelState()
{
var sortDescriptionClones = Enumerable.ToList(Enumerable.Select(this.SortDescriptions, (l) => (SortDescription)l.Clone()));
var rowGroupDescriptionClones = Enumerable.ToList(Enumerable.Select(this.RowGroupDescriptions, (l) => (PropertyGroupDescriptionBase)l.Clone()));
var columnGroupDescriptionClones = Enumerable.ToList(Enumerable.Select(this.ColumnGroupDescriptions, (l) => (PropertyGroupDescriptionBase)l.Clone()));
var aggregateDescriptionClones = Enumerable.ToList(Enumerable.Select(this.AggregateDescriptions, (l) => (PropertyAggregateDescriptionBase)l.Clone()));
var filterDescriptionClones = Enumerable.ToList(Enumerable.Select(this.FilterDescriptions, (l) => (PropertyFilterDescriptionBase)l.Clone()));
IComparer <object> sortComparer = sortDescriptionClones.Count > 0 ? new SortFieldComparer(sortDescriptionClones, null) : null;
var provider = new LocalSourceValueProvider()
{
RowGroupDescriptions = rowGroupDescriptionClones,
ColumnGroupDescriptions = columnGroupDescriptionClones,
AggregateDescriptions = aggregateDescriptionClones,
FilterDescriptions = filterDescriptionClones,
SortComparer = sortComparer,
GroupFactory = this.GroupFactory
};
ParallelState initalState = new ParallelState()
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
TaskScheduler = System.Threading.Tasks.TaskScheduler.Default,
RowGroupDescriptions = rowGroupDescriptionClones,
ColumnGroupDescriptions = columnGroupDescriptionClones,
AggregateDescriptions = aggregateDescriptionClones,
FilterDescriptions = filterDescriptionClones,
ValueProvider = provider,
DataView = this.DataView
};
return(initalState);
}
public ParallelState ReadParallelState()
{
var parallel = new ParallelState();
PushScope();
ReadTransitionalState(parallel);
PopScope();
return(parallel);
}
private void OnTreeEdge(TEdge e, ParallelState <TLocal> local)
{
var eh = this.TreeEdge;
if (eh != null)
{
eh(this, new ParallelEdgeEventArgs <TVertex, TEdge, TLocal>(e, local));
}
}
private void OnStartVertex(TVertex v, ParallelState <TLocal> local)
{
var eh = this.StartVertex;
if (eh != null)
{
eh(this, new ParallelVertexEventArgs <TVertex, TLocal>(v, local));
}
}
public override int Visit(ParallelState parallelState)
{
_currentContext.AssertIsValidInputPath(parallelState.InputPath);
_currentContext.AssertIsValidOutputPath(parallelState.OutputPath);
_currentContext.AssertIsValidResultPath(parallelState.ResultPath);
ValidateTransition(parallelState.Transition);
ValidateRetriers(parallelState.Retriers);
ValidateCatchers(parallelState.Catchers);
ValidateBranches(parallelState);
return(0);
}
private void EnqueueRoot(TVertex s, ParallelState <TLocal> local)
{
this.OnStartVertex(s, local);
var color = (GraphColor)Interlocked.Exchange(
ref this.vertexIndexedColors[this.vertexIndices[s]],
(int)GraphColor.Gray);
Contract.Assert(color == GraphColor.White);
OnDiscoverVertex(s, local);
this.vertexQueue.Enqueue(s);
}
private void ValidateParallelState(ValidationContext stateContext, ParallelState state)
{
var index = 0;
foreach (var branch in state.Branches)
{
new GraphValidator(stateContext.Branch(index),
new Dictionary <string, State>(),
branch.StartAt,
branch.States).Validate();
index++;
}
}
private void VisitLevel(ParallelState <TLocal> local)
{
var u = this.vertexQueue.Dequeue();
this.OnExamineVertex(u, local);
Parallel.ForEach <TEdge, TLocal>(
this.VisitedGraph.OutEdges(u),
this.CreateLocal,
(e, i, childLocal) => this.VisitChildren(e, childLocal),
this.OnLocalFinalized
);
this.OnFinishVertex(u, local);
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue,
JsonSerializer serializer)
{
var obj = JObject.Load(reader);
var type = obj["Type"].Value <string>();
State.IBuilder <State> result = null;
switch (Enum.Parse(typeof(StateType), type))
{
case StateType.Choice:
result = ChoiceState.GetBuilder();
break;
case StateType.Fail:
result = FailState.GetBuilder();
break;
case StateType.Parallel:
result = ParallelState.GetBuilder();
break;
case StateType.Pass:
result = PassState.GetBuilder();
break;
case StateType.Succeed:
result = SucceedState.GetBuilder();
break;
case StateType.Task:
result = TaskState.GetBuilder();
break;
case StateType.Wait:
result = WaitState.GetBuilder();
break;
case StateType.Map:
result = MapState.GetBuilder();
break;
}
serializer.Populate(obj.CreateReader(), result);
return(result);
}
private void VisitChildren(TEdge e, ParallelState <TLocal> local)
{
TVertex v = e.Target;
OnExamineEdge(e, local);
int vIndex = this.vertexIndices[v];
var vColor = (GraphColor)Interlocked.CompareExchange(
ref this.vertexIndexedColors[vIndex],
(int)GraphColor.Gray,
(int)GraphColor.White);
if (vColor == GraphColor.White)
{
this.OnTreeEdge(e, local);
this.OnDiscoverVertex(v, local);
this.vertexQueue.Enqueue(v);
}
}
private void ValidateBranches(ParallelState parallelState)
{
_currentContext.AssertNotEmpty(parallelState.Branches, PropertyNames.BRANCHES);
var index = 0;
foreach (var branch in parallelState.Branches)
{
var branchContext = _currentContext.Branch(index);
ValidateStates(branchContext, branch.States);
_currentContext.AssertStringNotEmpty(branch.StartAt, PropertyNames.START_AT);
if (!branch.States.ContainsKey(branch.StartAt))
{
_problemReporter.Report(new Problem(branchContext, $"{PropertyNames.START_AT} references a non existent state."));
}
index++;
}
}
private IEnumerable <int> ParseParallel(ParallelState state)
{
using (state)
{
foreach (var batch in state.GetBatches())
{
// If the collation of rows happened correctly, this should have a precise value.
Contracts.Assert(batch.Total == _total + 1);
_total = batch.Total - 1;
for (int irow = batch.IrowMin; irow < batch.IrowLim; irow++)
{
++_total;
yield return(irow);
}
}
if (state.ParsingException != null)
{
throw Ch.ExceptDecode(state.ParsingException,
"Parsing failed with an exception: {0}", state.ParsingException.Message);
}
}
}
public virtual void RebuildCube(ParallelState state)
{
this.ActionOnRebuildCube(state);
}
// Note that we don't filter out rows with parsing issues since it's not acceptable to
// produce a different set of rows when subsetting columns. Any parsing errors need to be
// translated to NaN, not result in skipping the row. We should produce some diagnostics
// to alert the user to the issues.
private Cursor(TextLoader parent, ParseStats stats, bool[] active, LineReader reader, int srcNeeded, int cthd)
: base(parent._host)
{
Ch.Assert(active == null || active.Length == parent._bindings.Infos.Length);
Ch.AssertValue(reader);
Ch.AssertValue(stats);
Ch.Assert(srcNeeded >= 0);
Ch.Assert(cthd > 0);
_total = -1;
_batch = -1;
_bindings = parent._bindings;
_parser = parent._parser;
_active = active;
_reader = reader;
_stats = stats;
_srcNeeded = srcNeeded;
ParallelState state = null;
if (cthd > 1)
{
state = new ParallelState(this, out _rows, cthd);
}
else
{
_rows = _parser.CreateRowSet(_stats, 1, _active);
}
try
{
_getters = new Delegate[_bindings.Infos.Length];
for (int i = 0; i < _getters.Length; i++)
{
if (_active != null && !_active[i])
{
continue;
}
ColumnPipe v = _rows.Pipes[i];
Ch.Assert(v != null);
_getters[i] = v.GetGetter();
Ch.Assert(_getters[i] != null);
}
if (state != null)
{
_ator = ParseParallel(state).GetEnumerator();
state = null;
}
else
{
_ator = ParseSequential().GetEnumerator();
}
}
finally
{
if (state != null)
{
state.Dispose();
}
}
}
public ParallelEventArgs(ParallelState <TLocal> local)
{
this.local = local;
}
public virtual void Clear(ParallelState state)
{
}
public object GetStartState()
{
var createCollection =
new AmazonStateTask <IndexAllFacesRequest, AmazonRekognitionClient, CreateCollectionRequest,
CreateCollectionResponse>()
.SetParameters(i => new CreateCollectionRequest
{
CollectionId = i.Collection,
Tags = new Dictionary <string, string>
{
{ "App", "Test" }
}
}
);
var waitState = new WaitState();
var getCollection =
new AmazonStateTask <IndexAllFacesRequest, AmazonRekognitionClient, DescribeCollectionRequest,
DescribeCollectionResponse>()
.SetParameters(i => new DescribeCollectionRequest
{
CollectionId = i.Collection
})
.AddErrorHandler(typeof(ResourceNotFoundException), new ErrorHandler
{
FallbackState = createCollection
});
var getObject = new AmazonStateTask <GetObjectRequest, AmazonS3Client, GetObjectRequest, GetObjectResponse>()
.SetParameters(i => i)
.Catch(typeof(Amazon.DynamoDBv2.Model.ResourceNotFoundException), createCollection)
.SetNextState(new SuccessState {
Name = "Done"
});
var getAllObjects = new MapState <ListObjectsV2Response, S3Object, GetObjectRequest>(getObject)
.SetIterator(i => i.S3Objects)
.SetMapping(i => new GetObjectRequest
{
BucketName = i.BucketName,
Key = i.Key
});
var listObjects =
new AmazonStateTask <IndexAllFacesRequest, AmazonS3Client, ListObjectsV2Request, ListObjectsV2Response>()
.SetParameters(i => new ListObjectsV2Request
{
BucketName = i.Bucket
})
.AddErrorHandler(typeof(AmazonS3Exception), new ErrorHandler())
.SetNextState(getAllObjects);
var getItem =
new AmazonStateTask <IndexAllFacesRequest, AmazonDynamoDBClient, GetItemRequest, GetItemResponse>()
.SetParameters(i => new GetItemRequest
{
TableName = i.TableName,
Key = new Dictionary <string, AttributeValue>
{
{ "Partition", new AttributeValue(i.Bucket) }
}
}).AddErrorHandler(typeof(Amazon.DynamoDBv2.Model.ResourceNotFoundException), new ErrorHandler {
FallbackState = new FailState()
});
var passState = new PassState(getItem)
{
};
var choiceState = new ChoiceSate <IndexAllFacesRequest>(passState)
.AddRule(i => i.Bucket.StartsWith("node") ||
i.TableName != "Something", waitState)
.AddRule(i => i.Collection == null, getItem);
var parallelTask = new ParallelState()
.AddState(getCollection)
.AddState(listObjects)
.AddState(getItem)
.SetNextState(new SuccessState());
return(choiceState);
return(parallelTask);
}
public ParallelVertexEventArgs(TVertex vertex, ParallelState <TLocal> local)
: base(vertex)
{
this.local = local;
}
public ParallelEdgeEventArgs(TEdge edge, ParallelState <TLocal> local)
: base(edge)
{
this.local = local;
}
public virtual T Visit(ParallelState parallelState)
{
return(default(T));
}
