/// <summary> /// Start displaying information about this stage. /// </summary> /// <param name="stage">Job stage to display.</param> private void SetStage(DryadLinqJobStage stage) { this.Status("Loading stage " + stage.Name + " information...", StatusKind.LongOp); this.EnableStageFiltering(true); this.stageHeaderData.RaiseListChangedEvents = false; this.currentStage = stage; this.currentTable = null; // stageData is populated by the selectionChanged event handler for the stageHeader if (this.ShowingStageOrTable != KindOfStageShown.Stage) { // if we are changing the nature of the datasource (from table to stage) we need to do some work this.ShowingStageOrTable = KindOfStageShown.Stage; this.dataGridView_stageContents.SuspendLayout(); BindingListSortable<ExecutedVertexInstance> empty = new BindingListSortable<ExecutedVertexInstance>(); // bind to an empty list to make the property changes fast this.dataGridView_stageContents.DataSource = empty; this.dataGridView_stageHeader.DataSource = this.stageHeaderData; this.SetDataGridViewColumnsSize(this.dataGridView_stageHeader); DataGridViewColumnCollection columns = this.dataGridView_stageContents.Columns; // ReSharper disable PossibleNullReferenceException columns["Name"].DisplayIndex = 0; columns["Start"].DefaultCellStyle.Format = "T"; columns["Start"].DisplayIndex = 1; columns["End"].DefaultCellStyle.Format = "T"; columns["End"].DisplayIndex = 2; columns["RunningTime"].DefaultCellStyle.Format = "g"; columns["RunningTime"].DisplayIndex = 3; columns["DataRead"].DefaultCellStyle.Format = "N0"; columns["DataWritten"].DefaultCellStyle.Format = "N0"; string[] invisibleColumns = { "IsManager", /* "ProcessIdentifier",*/ "Number", "WorkDirectory", "ErrorString", "StdoutFile", "LogFilesPattern", "LogDirectory", "UniqueID", "InputChannels", "OutputChannels", "CreationTime", "StartCommandTime", "VertexScheduleTime", "StageName", "DataRead", "DataWritten", "ExitCode", "VertexScheduleTime", "StartCommandTime", "VertexIsCompleted" }; foreach (string s in invisibleColumns) { if (s == "DataRead" || s == "DataWritten") continue; columns[s].Visible = false; } columns["Version"].HeaderText = "v."; this.SetDataGridViewColumnsSize(this.dataGridView_stageContents); // bind to the actual data this.dataGridView_stageContents.DataSource = this.stageData; this.dataGridView_stageContents.ResumeLayout(); } else { this.dataGridView_stageContents.Columns["DataRead"].Visible = true; this.dataGridView_stageContents.Columns["DataWritten"].Visible = true; } // ReSharper restore PossibleNullReferenceException this.stageHeaderData.Clear(); if (this.staticPlan != null) { DryadJobStaticPlan.Stage s = this.staticPlan.GetStageByName(stage.Name); if (s != null) { this.textBox_stageCode.Lines = s.Code; stage.StaticVertexCount = s.Replication; } else { this.textBox_stageCode.Lines = null; } } this.label_stage.Text = "Stage: " + stage.Name; this.stagePropertyEnumerator.Data = stage; this.stagePropertyEnumerator.PopulateWithProperties(this.stageHeaderData); this.stageHeaderData.RaiseListChangedEvents = true; this.stageHeaderData.ResetBindings(); // display by default the work directory of the job manager if nothing is displayed if (stage.Name == "JobManager" && this.comboBox_vertexInformation.Text == "" && Environment.UserName == "jcurrey") { this.comboBox_vertexInformation.Text = "work dir"; } this.Status("OK", StatusKind.OK); }
/// <summary> /// Start displaying information about this table. /// </summary> /// <param name="tableinfo">Table information to display.</param> private void SetTable(StaticPartitionedTableInformation tableinfo) { this.EnableStageFiltering(false); this.stageHeaderData.RaiseListChangedEvents = false; this.currentStage = null; this.currentTable = tableinfo; this.tableHeaderData.Clear(); if (this.ShowingStageOrTable != KindOfStageShown.Table) { this.dataGridView_stageHeader.DataSource = this.tableHeaderData; this.dataGridView_stageContents.DataSource = this.tablePartitionsData; // ReSharper disable PossibleNullReferenceException this.dataGridView_stageContents.Columns["PartitionSize"].DefaultCellStyle.Format = "N0"; // ReSharper restore PossibleNullReferenceException this.SetDataGridViewColumnsSize(this.dataGridView_stageContents); this.ShowingStageOrTable = KindOfStageShown.Table; } if (tableinfo.Code != null) { this.textBox_stageCode.Lines = tableinfo.Code; } else { this.textBox_stageCode.Lines = null; } this.label_stage.Text = "Table: " + tableinfo.Header; this.tablePropertyEnumerator.Data = tableinfo; this.tablePropertyEnumerator.PopulateWithProperties(this.tableHeaderData); this.PopulateTablePartitions(); this.stageHeaderData.RaiseListChangedEvents = true; this.stageHeaderData.ResetBindings(); if (tableinfo.Error == "") this.Status("OK", StatusKind.OK); }
/// <summary> /// Create a class representing the set of edges between two stages in the job plan. /// These collectively look like a partitioned table. /// If the source stage has multiple outputs there is not enough information to return meaningful information. /// </summary> /// <param name="job">Job whose slice we are displaying.</param> /// <param name="plan">Static plan of the job.</param> /// <param name="source">Stage in the job which produces the data.</param> /// <param name="status">Delegate used to report errors.</param> /// <param name="showCancelled">If true include cancelled vertices.</param> public static StaticPartitionedTableInformation StageOutput( DryadLinqJobInfo job, DryadJobStaticPlan plan, DryadJobStaticPlan.Stage source, StatusReporter status, bool showCancelled) { string header = "Output of " + source.Name; // First check whether in the static plan this is virtual while (source.IsTee) { var sourceInputs = plan.GetStageConnections(source, true).ToList(); if (sourceInputs.Count() != 1) throw new DryadException("Unexpected number of inputs for stage " + source.Name); source = sourceInputs.First().From; } // If we reached the input return information about that input if (source.IsInput) { status("Scanning " + source.Name, StatusKind.LongOp); StaticPartitionedTableInformation result = new StaticPartitionedTableInformation(job.ClusterConfiguration, source.UriType, source.Uri, source.Code, status); result.Header = "Output of " + header; result.constructorArguments = new SaveConstructorArguments { code = null, source = source, plan = plan }; return result; } else { StaticPartitionedTableInformation result = new StaticPartitionedTableInformation(); result.Name = "Output of vertices in stage " + source.Name; result.Header = "Output of " + header; result.constructorArguments = new SaveConstructorArguments { code = null, source = source, plan = plan }; // Check whether this stage has multiple outputs; this can only happen for 'Fork' operators. var destinations = plan.GetStageConnections(source, false); if (destinations.Count() > 1) { result.Error = "Cannot provide information about one of multiple outputs of a stage."; return result; } DryadLinqJobStage stage = job.GetStage(source.Name); if (stage == null) { result.Error = "There is no information about the output of stage " + source.Name; return result; } result.EstimatedSize = 0; result.PartitionCount = stage.TotalInitiatedVertices; int count = 0; foreach (ExecutedVertexInstance vi in stage.Vertices) { if (vi.State == ExecutedVertexInstance.VertexState.Successful || vi.State == ExecutedVertexInstance.VertexState.Failed || (showCancelled && vi.State == ExecutedVertexInstance.VertexState.Cancelled)) { StaticPartitionInformation spi = new StaticPartitionInformation(count++, vi.DataWritten, vi.Name + " v." + vi.Version); result.AddPartition(spi); if (vi.DataWritten != -1) result.EstimatedSize += vi.DataWritten; } } return result; } }
/// <summary> /// User clicked on the job plan; figure out what to display. /// </summary> /// <param name="xcoord">X coordinate of click, in pixels.</param> /// <param name="ycoord">Y coordinate of click, in pixels.</param> /// <returns>If true, the event has been completely handled; else it will be handled by the default handler as well.</returns> private void ClickOnJobPlan(int xcoord, int ycoord) { if (this.planVisible == PlanVisible.None) return; if (this.planVisible == PlanVisible.Static) { DryadJobStaticPlan.Stage stage; object o = this.graphViewer.GetObjectAt(xcoord, ycoord); var dEdge = o as DEdge; if (dEdge != null) { Msagl.Drawing.Edge edge = (dEdge).DrawingEdge; DryadJobStaticPlan.Connection connection = (DryadJobStaticPlan.Connection)edge.UserData; StaticPartitionedTableInformation info = StaticPartitionedTableInformation.StageOutput(this.Job, this.staticPlan, connection.From, this.Status, !this.hideCancelledVerticesToolStripMenuItem.Checked); this.SetTable(info); return; } var dNode = o as DNode; if (dNode != null) { Msagl.Drawing.Node node = dNode.DrawingNode; stage = (DryadJobStaticPlan.Stage)node.UserData; } else { this.SetNoStageOrTable("", false); return; } if (stage.IsVirtual) { if (stage.IsInput || (this.Job.Summary.Status != ClusterJobInformation.ClusterJobStatus.Running && stage.IsOutput)) // can't display output tables if the job is not ready yet { // some input or output tables have associated processes; if that's true, display these instead of the table. DryadLinqJobStage istage = this.Job.GetStage(stage.Name); if (istage != null && istage.TotalInitiatedVertices != 0) { this.SetStage(istage); } else { this.Status("Reading table information", StatusKind.LongOp); StaticPartitionedTableInformation info = new StaticPartitionedTableInformation(this.Job.ClusterConfiguration, stage.UriType, stage.Uri, stage.Code, this.Status); this.SetTable(info); } } else if (stage.IsTee) { StaticPartitionedTableInformation info = StaticPartitionedTableInformation.StageOutput(this.Job, this.staticPlan, stage, this.Status, !this.hideCancelledVerticesToolStripMenuItem.Checked); this.SetTable(info); return; } else { this.SetNoStageOrTable(stage.Name, stage.IsOutput); } return; } else { DryadLinqJobStage executedstage = this.Job.GetStage(stage.Name); if (executedstage != null) this.SetStage(executedstage); } } else if (this.planVisible == PlanVisible.Dynamic) { // dynamic plan visible double xo, yo; this.planDrawSurface.ScreenToUser(xcoord, ycoord, out xo, out yo); GraphLayout.GraphNode node = this.dynamicPlanLayout.FindNode(xo, yo); this.dynamicPlanLayout.ClearSelectedNodes(); if (node != null) { node.Selected = true; string stageName = node.Stage; DryadLinqJobStage executedstage = this.Job.GetStage(stageName); if (executedstage != null) this.SetStage(executedstage); } else { this.SetNoStageOrTable("", false); } this.DrawQueryPlan(); // refresh the image } else if (this.planVisible == PlanVisible.Schedule) { double xo, yo; this.planDrawSurface.ScreenToUser(xcoord, ycoord, out xo, out yo); ExecutedVertexInstance vertex = this.jobSchedule.FindVertex(xo, yo); this.SelectVertex(vertex); } return; }
/// <summary> /// Create a class representing the set of edges between two stages in the job plan. /// These collectively look like a partitioned table. /// If the source stage has multiple outputs there is not enough information to return meaningful information. /// </summary> /// <param name="job">Job whose slice we are displaying.</param> /// <param name="plan">Static plan of the job.</param> /// <param name="source">Stage in the job which produces the data.</param> /// <param name="status">Delegate used to report errors.</param> /// <param name="showCancelled">If true include cancelled vertices.</param> public static StaticPartitionedTableInformation StageOutput( DryadLinqJobInfo job, DryadJobStaticPlan plan, DryadJobStaticPlan.Stage source, StatusReporter status, bool showCancelled) { string header = "Output of " + source.Name; // First check whether in the static plan this is virtual while (source.IsTee) { var sourceInputs = plan.GetStageConnections(source, true).ToList(); if (sourceInputs.Count() != 1) { throw new DryadException("Unexpected number of inputs for stage " + source.Name); } source = sourceInputs.First().From; } // If we reached the input return information about that input if (source.IsInput) { status("Scanning " + source.Name, StatusKind.LongOp); StaticPartitionedTableInformation result = new StaticPartitionedTableInformation(job.ClusterConfiguration, source.UriType, source.Uri, source.Code, status); result.Header = "Output of " + header; result.constructorArguments = new SaveConstructorArguments { code = null, source = source, plan = plan }; return(result); } else { StaticPartitionedTableInformation result = new StaticPartitionedTableInformation(); result.Name = "Output of vertices in stage " + source.Name; result.Header = "Output of " + header; result.constructorArguments = new SaveConstructorArguments { code = null, source = source, plan = plan }; // Check whether this stage has multiple outputs; this can only happen for 'Fork' operators. var destinations = plan.GetStageConnections(source, false); if (destinations.Count() > 1) { result.Error = "Cannot provide information about one of multiple outputs of a stage."; return(result); } DryadLinqJobStage stage = job.GetStage(source.Name); if (stage == null) { result.Error = "There is no information about the output of stage " + source.Name; return(result); } result.EstimatedSize = 0; result.PartitionCount = stage.TotalInitiatedVertices; int count = 0; foreach (ExecutedVertexInstance vi in stage.Vertices) { if (vi.State == ExecutedVertexInstance.VertexState.Successful || vi.State == ExecutedVertexInstance.VertexState.Failed || (showCancelled && vi.State == ExecutedVertexInstance.VertexState.Cancelled)) { StaticPartitionInformation spi = new StaticPartitionInformation(count++, vi.DataWritten, vi.Name + " v." + vi.Version); result.AddPartition(spi); if (vi.DataWritten != -1) { result.EstimatedSize += vi.DataWritten; } } } return(result); } }