public override async Task ExecuteWorkAsync() { if (communicator.Rank == 0) { await base.ExecuteWorkAsync(); return; } await nodesManager.CreateWorkerNodesAsync(); }
private async Task WorkLoopAsync() { var nodes = await nodesManager.CreateWorkerNodesAsync(); workerNodes = nodes.ToArray(); var nodesEnumerator = nodesStream().GetEnumerator(); nodesEnumerator.MoveNext(); var assignments = assignmentsStream(); foreach (var assignmentFunction in assignments) { var assignedNode = nodesEnumerator.Current; var advanceNode = await assignmentFunction(assignedNode); if (advanceNode) { nodesEnumerator.MoveNext(); } } await nodesManager.StopWorkerNodesAsync(); IEnumerable <Func <int, Task <bool> > > assignmentsStream() { while (filepathsToTokenize.Any()) { var nextFilePath = filepathsToTokenize.Dequeue(); yield return((nodeId) => { var assignment = new TokenizationAssignment(nodeId, nextFilePath); messageSink.PostTokenizationAssignment(assignment); return Task.FromResult(true); }); } yield return((nodeId) => tokenizationDoneTCS.Task.ContinueWith(_ => false)); foreach (var node in workerNodes) { yield return((nodeId) => { var assignment = new ConfigureNormalizationAssignment(nodeId, generatedDocuments.Count(), termDocumentAppearances); messageSink.PostConfigureNormalizationAssignment(assignment); return Task.FromResult(true); }); } while (documentsToNormalize.Any()) { var nextDocument = documentsToNormalize.Dequeue(); yield return((nodeId) => { var assignment = new NormalizationAssignment(nodeId, nextDocument); messageSink.PostNormalizationAssignment(assignment); return Task.FromResult(true); }); } yield break; } IEnumerable <int> nodesStream() { var currentNodeId = 0; while (true) { currentNodeId++; if (currentNodeId > workerNodes.Length) { currentNodeId = 0; } yield return(currentNodeId); } } }