Exemple #1
0
        private async Task WorkLoopAsync()
        {
            var nodes = await nodesManager.CreateWorkerNodesAsync();

            workerNodes = nodes.ToArray();

            var nodesEnumerator = nodesStream().GetEnumerator();

            nodesEnumerator.MoveNext();

            var assignments = assignmentsStream();

            foreach (var assignmentFunction in assignments)
            {
                var assignedNode = nodesEnumerator.Current;
                var advanceNode  = await assignmentFunction(assignedNode);

                if (advanceNode)
                {
                    nodesEnumerator.MoveNext();
                }
            }

            await nodesManager.StopWorkerNodesAsync();

            IEnumerable <Func <int, Task <bool> > > assignmentsStream()
            {
                while (filepathsToTokenize.Any())
                {
                    var nextFilePath = filepathsToTokenize.Dequeue();

                    yield return((nodeId) => {
                        var assignment = new TokenizationAssignment(nodeId, nextFilePath);
                        messageSink.PostTokenizationAssignment(assignment);

                        return Task.FromResult(true);
                    });
                }

                yield return((nodeId) => tokenizationDoneTCS.Task.ContinueWith(_ => false));

                foreach (var node in workerNodes)
                {
                    yield return((nodeId) => {
                        var assignment = new ConfigureNormalizationAssignment(nodeId, generatedDocuments.Count(), termDocumentAppearances);
                        messageSink.PostConfigureNormalizationAssignment(assignment);

                        return Task.FromResult(true);
                    });
                }

                while (documentsToNormalize.Any())
                {
                    var nextDocument = documentsToNormalize.Dequeue();

                    yield return((nodeId) => {
                        var assignment = new NormalizationAssignment(nodeId, nextDocument);
                        messageSink.PostNormalizationAssignment(assignment);

                        return Task.FromResult(true);
                    });
                }

                yield break;
            }

            IEnumerable <int> nodesStream()
            {
                var currentNodeId = 0;

                while (true)
                {
                    currentNodeId++;
                    if (currentNodeId > workerNodes.Length)
                    {
                        currentNodeId = 0;
                    }

                    yield return(currentNodeId);
                }
            }
        }