/// <summary> /// Create a class to diagnose the problems of a vertex. /// </summary> /// <param name="vertex">Vertex to diagnose.</param> /// <param name="job">Job containing the vertex.</param> /// <param name="plan">Plan of the executed job.</param> /// <param name="manager">Communication manager.</param> protected VertexFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, ExecutedVertexInstance vertex, CommManager manager) : base(job, plan, manager) { this.Job = job; this.Vertex = vertex; // ReSharper disable once DoNotCallOverridableMethodsInConstructor this.stackTraceFile = "dryadLinqStackTrace.txt"; }
/// <summary> /// Create a FailureDiagnosis object. /// </summary> /// <param name="job">Job being diagnosed.</param> /// <param name="plan">Static plan of the job.</param> /// <param name="manager">Communication manager.</param> protected FailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, CommManager manager) { this.Job = job; this.StaticPlan = plan; this.Manager = manager; this.Summary = job.Summary; this.cluster = job.ClusterConfiguration; }
/// <summary> /// Try to find the job information from cluster and summary. /// </summary> /// <param name="manager">Communication manager.</param> protected void FindJobInfo(CommManager manager) { DryadLinqJobInfo jobinfo = DryadLinqJobInfo.CreateDryadLinqJobInfo(this.cluster, this.Summary, true, manager); if (jobinfo == null) { manager.Status("Cannot collect information for " + Summary.ShortName() + " to diagnose", StatusKind.Error); return; } this.Job = jobinfo; this.StaticPlan = JobObjectModel.DryadJobStaticPlan.CreatePlan(jobinfo, manager); }
/// <summary> /// Create a VertexFailureDiagnosis of the appropriate type. /// </summary> /// <param name="vertex">Vertex to diagnose.</param> /// <param name="job">Job containing the vertex.</param> /// <param name="manager">Communication manager.</param> /// <returns>A subclass of VertexFailureDiagnosis.</returns> /// <param name="plan">Plan of the executed job.</param> public static VertexFailureDiagnosis CreateVertexFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, ExecutedVertexInstance vertex, CommManager manager) { ClusterConfiguration config = job.ClusterConfiguration; if (config is CacheClusterConfiguration) { config = (config as CacheClusterConfiguration).ActualConfig(job.Summary); } throw new InvalidOperationException("Config of type " + config.TypeOfCluster + " not handled"); }
/// <summary> /// Create a suitable Job Failure diagnosis object for the job being analyzed. /// </summary> /// <param name="job">Job to diagnose.</param> /// <param name="manager">Communication manager.</param> /// <returns>A subclass of JobFailureDiagnosis with the type appropriate for the job.</returns> /// <param name="plan">Plan of the job being diagnosed.</param> public static JobFailureDiagnosis CreateJobFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, CommManager manager) { ClusterConfiguration config = job.ClusterConfiguration; if (config is CacheClusterConfiguration) { config = (config as CacheClusterConfiguration).ActualConfig(job.Summary); } throw new InvalidOperationException("Configuration of type " + config.TypeOfCluster + " not supported for diagnosis"); }
/// <summary> /// Create a class to diagnose the problems of a job. /// </summary> /// <param name="job">Job to diagnose.</param> /// <param name="plan">Plan of the diagnosed job.</param> /// <param name="manager">Communication manager.</param> protected JobFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, CommManager manager) : base(job, plan, manager) { this.diagnosisLog = new DiagnosisLog(job, job.Summary); this.jobManager = this.Job.ManagerVertex; }
/// <summary> /// Create a class representing the set of edges between two stages in the job plan. /// These collectively look like a partitioned table. /// If the source stage has multiple outputs there is not enough information to return meaningful information. /// </summary> /// <param name="job">Job whose slice we are displaying.</param> /// <param name="plan">Static plan of the job.</param> /// <param name="source">Stage in the job which produces the data.</param> /// <param name="status">Delegate used to report errors.</param> /// <param name="showCancelled">If true include cancelled vertices.</param> public static StaticPartitionedTableInformation StageOutput( DryadLinqJobInfo job, DryadJobStaticPlan plan, DryadJobStaticPlan.Stage source, StatusReporter status, bool showCancelled) { string header = "Output of " + source.Name; // First check whether in the static plan this is virtual while (source.IsTee) { var sourceInputs = plan.GetStageConnections(source, true).ToList(); if (sourceInputs.Count() != 1) { throw new DryadException("Unexpected number of inputs for stage " + source.Name); } source = sourceInputs.First().From; } // If we reached the input return information about that input if (source.IsInput) { status("Scanning " + source.Name, StatusKind.LongOp); StaticPartitionedTableInformation result = new StaticPartitionedTableInformation(job.ClusterConfiguration, source.UriType, source.Uri, source.Code, status); result.Header = "Output of " + header; result.constructorArguments = new SaveConstructorArguments { code = null, source = source, plan = plan }; return(result); } else { StaticPartitionedTableInformation result = new StaticPartitionedTableInformation(); result.Name = "Output of vertices in stage " + source.Name; result.Header = "Output of " + header; result.constructorArguments = new SaveConstructorArguments { code = null, source = source, plan = plan }; // Check whether this stage has multiple outputs; this can only happen for 'Fork' operators. var destinations = plan.GetStageConnections(source, false); if (destinations.Count() > 1) { result.Error = "Cannot provide information about one of multiple outputs of a stage."; return(result); } DryadLinqJobStage stage = job.GetStage(source.Name); if (stage == null) { result.Error = "There is no information about the output of stage " + source.Name; return(result); } result.EstimatedSize = 0; result.PartitionCount = stage.TotalInitiatedVertices; int count = 0; foreach (ExecutedVertexInstance vi in stage.Vertices) { if (vi.State == ExecutedVertexInstance.VertexState.Successful || vi.State == ExecutedVertexInstance.VertexState.Failed || (showCancelled && vi.State == ExecutedVertexInstance.VertexState.Cancelled)) { StaticPartitionInformation spi = new StaticPartitionInformation(count++, vi.DataWritten, vi.Name + " v." + vi.Version); result.AddPartition(spi); if (vi.DataWritten != -1) { result.EstimatedSize += vi.DataWritten; } } } return(result); } }