private void ProcessWorkerStatistics(WorkerStatistics workerStatistics) { PipelineSection pipelineSection = FindPipelineSection(workerStatistics.WorkerBadge.SectionName); if (pipelineSection == null) { return; } WorkerStatus workerStatus; bool found = pipelineSection.WorkerStatuses.TryGetValue(workerStatistics.WorkerBadge.WorkerId, out workerStatus); if (found) { workerStatus.WorkerStatistics = workerStatistics; } else { workerStatus = new WorkerStatus() { WorkerStatistics = workerStatistics }; pipelineSection.WorkerStatuses.Add(workerStatistics.WorkerBadge.WorkerId, workerStatus); } StatisticsAvailable = true; //Tracer.Trace("Worker {0} stat: processed {1} documents, NetTime {2:0.#} seconds, NetSpeed {3:0.##} docs/sec", // workerStatistics.WorkerId, // workerStatistics.ProcessedDocuments, // workerStatistics.WorkTimeNet.TotalSeconds, // workerStatistics.PersonalSpeedNet // ); }
public void Build(PipelineBuildOrder pipelineBuildOrder) { string machineName = Environment.MachineName; PipelineType = pipelineBuildOrder.PipelineType; PipelineId = pipelineBuildOrder.PipelineId.ToString(CultureInfo.InvariantCulture); ReportPipeName = new ReportPipeName(machineName, PipelineType, PipelineId); using (ReportPipe = new Pipe(ReportPipeName)) { ReportPipe.Create(); } uint orderNumber = 0; foreach (var rolePlan in pipelineBuildOrder.RolePlans) { PipelineSection pipelineSection = new PipelineSection(machineName, orderNumber, PipelineType, PipelineId, rolePlan); PipelineSections.Add(pipelineSection); if (rolePlan.Name == "Log") { LogDataPipeName = new DataPipeName(machineName, PipelineType, PipelineId, "Log" /* section name*/); } orderNumber++; } PipelineStatus.PipelineState = PipelineState.Built; }
protected override void PeriodicPipelineServicingHook() { base.PeriodicPipelineServicingHook(); if (CurrentContinuation > NumberOfContinuations) { return; } if (PipelineStatus.PipelineState < PipelineState.FirstWorkerCompleted) { Tracer.Warning("FIRST WORKER HAS NOT COMPLETED YET!"); return; } PipelineSection triggerSection = PipelineSections.Find(pipelineSection => pipelineSection.OrderNumber == 1); if (triggerSection == null) { return; } if (triggerSection.DataPipeName != null && triggerSection.DataPipeName.Count > 0) { Tracer.Warning("SECOND SECTION IS STILL BUSY!"); return; } Tracer.Warning("ACTION GOES HERE!"); CurrentContinuation++; }
protected void ProcessWorkerStateChangeReports(WorkerStateChangedMessage message) { PipelineSection pipelineSection = FindPipelineSection(message.WorkerBadge.SectionName); if (pipelineSection == null) { return; } WorkerStatus workerStatus; bool found = pipelineSection.WorkerStatuses.TryGetValue(message.WorkerBadge.WorkerId, out workerStatus); if (found) { workerStatus.WorkerState = message.WorkerState; } else { workerStatus = new WorkerStatus() { WorkerBadge = message.WorkerBadge, WorkerState = message.WorkerState }; pipelineSection.WorkerStatuses.Add(message.WorkerBadge.WorkerId, workerStatus); Tracer.Trace("Pipeline {0}, section {1} added new worker {2} with state {3}", PipelineId, pipelineSection.Name, message.WorkerBadge.WorkerId, workerStatus.WorkerState); } // Debug //Tracer.Warning("Worker " + message.WorkerBadge.WorkerId + " reported state " + message.WorkerState); if (pipelineSection.OrderNumber == 0 && message.WorkerState == WorkerState.Started) { Tracer.Info("Pipeline initialization completed in {0:0.##} seconds. First worker started filling up the pipe.", _runTime.Elapsed.TotalSeconds); } // Handling first worker completion if (pipelineSection.OrderNumber == 0 && message.WorkerState == WorkerState.Completed) { // First worker reported that it generated at least one message to the pipe. PipelineStatus.PipelineState = PipelineState.FirstWorkerCompleted; CheckPipelineForCompletion(); } // Handling worker quit if (message.WorkerState == WorkerState.Quit) { if (pipelineSection.WorkersNotQuit == 0) { long remainingWorkers = PipelineSections.Aggregate <PipelineSection, long>(0, (current, curPipelineSection) => current + curPipelineSection.WorkersNotQuit); if (remainingWorkers == 0) { PipelineStatus.PipelineState = PipelineState.AllWorkersQuit; } } } }
public List <WorkRequest> GenerateWorkRequests() { try { List <WorkRequest> workRequests = new List <WorkRequest>(); foreach (PipelineSection pipelineSection in PipelineSections) { WorkRequest workRequest = new WorkRequest ( pipelineSection.Name, pipelineSection.DataPipeName, pipelineSection.HiringPipeName, LogDataPipeName, ReportPipe.Name as ReportPipeName, pipelineSection.RoleType, PipelineType, PipelineId, null ) { OutputSections = new List <WorkRequest.OutputSection>() }; foreach (string outputSectionName in pipelineSection.OutputSectionNames) { PipelineSection outputPipelineSection = PipelineSections.Find(item => item.Name == outputSectionName); workRequest.OutputSections.Add(new WorkRequest.OutputSection(outputSectionName, outputPipelineSection.DataPipeName)); } workRequests.Add(workRequest); } return(workRequests); } catch (Exception ex) { ex.AddDbgMsg("Work request generation failed.").Trace().Swallow(); return(null); // Calling method handles this gracefully. } }