/// <summary>
/// Start displaying information about this stage.
/// </summary>
/// <param name="stage">Job stage to display.</param>
private void SetStage(DryadLinqJobStage stage)
{
this.Status("Loading stage " + stage.Name + " information...", StatusKind.LongOp);
this.EnableStageFiltering(true);
this.stageHeaderData.RaiseListChangedEvents = false;
this.currentStage = stage;
this.currentTable = null;
// stageData is populated by the selectionChanged event handler for the stageHeader
if (this.ShowingStageOrTable != KindOfStageShown.Stage)
{
// if we are changing the nature of the datasource (from table to stage) we need to do some work
this.ShowingStageOrTable = KindOfStageShown.Stage;
this.dataGridView_stageContents.SuspendLayout();
BindingListSortable<ExecutedVertexInstance> empty = new BindingListSortable<ExecutedVertexInstance>();
// bind to an empty list to make the property changes fast
this.dataGridView_stageContents.DataSource = empty;
this.dataGridView_stageHeader.DataSource = this.stageHeaderData;
this.SetDataGridViewColumnsSize(this.dataGridView_stageHeader);
DataGridViewColumnCollection columns = this.dataGridView_stageContents.Columns;
// ReSharper disable PossibleNullReferenceException
columns["Name"].DisplayIndex = 0;
columns["Start"].DefaultCellStyle.Format = "T";
columns["Start"].DisplayIndex = 1;
columns["End"].DefaultCellStyle.Format = "T";
columns["End"].DisplayIndex = 2;
columns["RunningTime"].DefaultCellStyle.Format = "g";
columns["RunningTime"].DisplayIndex = 3;
columns["DataRead"].DefaultCellStyle.Format = "N0";
columns["DataWritten"].DefaultCellStyle.Format = "N0";
string[] invisibleColumns =
{
"IsManager", /* "ProcessIdentifier",*/ "Number", "WorkDirectory", "ErrorString",
"StdoutFile", "LogFilesPattern", "LogDirectory", "UniqueID", "InputChannels", "OutputChannels",
"CreationTime", "StartCommandTime", "VertexScheduleTime", "StageName",
"DataRead", "DataWritten", "ExitCode", "VertexScheduleTime", "StartCommandTime", "VertexIsCompleted"
};
foreach (string s in invisibleColumns)
{
if (s == "DataRead" || s == "DataWritten")
continue;
columns[s].Visible = false;
}
columns["Version"].HeaderText = "v.";
this.SetDataGridViewColumnsSize(this.dataGridView_stageContents);
// bind to the actual data
this.dataGridView_stageContents.DataSource = this.stageData;
this.dataGridView_stageContents.ResumeLayout();
}
else
{
this.dataGridView_stageContents.Columns["DataRead"].Visible = true;
this.dataGridView_stageContents.Columns["DataWritten"].Visible = true;
}
// ReSharper restore PossibleNullReferenceException
this.stageHeaderData.Clear();
if (this.staticPlan != null)
{
DryadJobStaticPlan.Stage s = this.staticPlan.GetStageByName(stage.Name);
if (s != null)
{
this.textBox_stageCode.Lines = s.Code;
stage.StaticVertexCount = s.Replication;
}
else
{
this.textBox_stageCode.Lines = null;
}
}
this.label_stage.Text = "Stage: " + stage.Name;
this.stagePropertyEnumerator.Data = stage;
this.stagePropertyEnumerator.PopulateWithProperties(this.stageHeaderData);
this.stageHeaderData.RaiseListChangedEvents = true;
this.stageHeaderData.ResetBindings();
// display by default the work directory of the job manager if nothing is displayed
if (stage.Name == "JobManager" && this.comboBox_vertexInformation.Text == "" && Environment.UserName == "jcurrey")
{
this.comboBox_vertexInformation.Text = "work dir";
}
this.Status("OK", StatusKind.OK);
}
/// <summary>
/// Start displaying information about this table.
/// </summary>
/// <param name="tableinfo">Table information to display.</param>
private void SetTable(StaticPartitionedTableInformation tableinfo)
{
this.EnableStageFiltering(false);
this.stageHeaderData.RaiseListChangedEvents = false;
this.currentStage = null;
this.currentTable = tableinfo;
this.tableHeaderData.Clear();
if (this.ShowingStageOrTable != KindOfStageShown.Table)
{
this.dataGridView_stageHeader.DataSource = this.tableHeaderData;
this.dataGridView_stageContents.DataSource = this.tablePartitionsData;
// ReSharper disable PossibleNullReferenceException
this.dataGridView_stageContents.Columns["PartitionSize"].DefaultCellStyle.Format = "N0";
// ReSharper restore PossibleNullReferenceException
this.SetDataGridViewColumnsSize(this.dataGridView_stageContents);
this.ShowingStageOrTable = KindOfStageShown.Table;
}
if (tableinfo.Code != null)
{
this.textBox_stageCode.Lines = tableinfo.Code;
}
else
{
this.textBox_stageCode.Lines = null;
}
this.label_stage.Text = "Table: " + tableinfo.Header;
this.tablePropertyEnumerator.Data = tableinfo;
this.tablePropertyEnumerator.PopulateWithProperties(this.tableHeaderData);
this.PopulateTablePartitions();
this.stageHeaderData.RaiseListChangedEvents = true;
this.stageHeaderData.ResetBindings();
if (tableinfo.Error == "")
this.Status("OK", StatusKind.OK);
}
/// <summary>
/// Create a class representing the set of edges between two stages in the job plan.
/// These collectively look like a partitioned table.
/// If the source stage has multiple outputs there is not enough information to return meaningful information.
/// </summary>
/// <param name="job">Job whose slice we are displaying.</param>
/// <param name="plan">Static plan of the job.</param>
/// <param name="source">Stage in the job which produces the data.</param>
/// <param name="status">Delegate used to report errors.</param>
/// <param name="showCancelled">If true include cancelled vertices.</param>
public static StaticPartitionedTableInformation StageOutput(
DryadLinqJobInfo job,
DryadJobStaticPlan plan,
DryadJobStaticPlan.Stage source,
StatusReporter status,
bool showCancelled)
{
string header = "Output of " + source.Name;
// First check whether in the static plan this is virtual
while (source.IsTee)
{
var sourceInputs = plan.GetStageConnections(source, true).ToList();
if (sourceInputs.Count() != 1)
throw new DryadException("Unexpected number of inputs for stage " + source.Name);
source = sourceInputs.First().From;
}
// If we reached the input return information about that input
if (source.IsInput)
{
status("Scanning " + source.Name, StatusKind.LongOp);
StaticPartitionedTableInformation result = new StaticPartitionedTableInformation(job.ClusterConfiguration, source.UriType, source.Uri, source.Code, status);
result.Header = "Output of " + header;
result.constructorArguments = new SaveConstructorArguments
{
code = null,
source = source,
plan = plan
};
return result;
}
else
{
StaticPartitionedTableInformation result = new StaticPartitionedTableInformation();
result.Name = "Output of vertices in stage " + source.Name;
result.Header = "Output of " + header;
result.constructorArguments = new SaveConstructorArguments
{
code = null,
source = source,
plan = plan
};
// Check whether this stage has multiple outputs; this can only happen for 'Fork' operators.
var destinations = plan.GetStageConnections(source, false);
if (destinations.Count() > 1)
{
result.Error = "Cannot provide information about one of multiple outputs of a stage.";
return result;
}
DryadLinqJobStage stage = job.GetStage(source.Name);
if (stage == null)
{
result.Error = "There is no information about the output of stage " + source.Name;
return result;
}
result.EstimatedSize = 0;
result.PartitionCount = stage.TotalInitiatedVertices;
int count = 0;
foreach (ExecutedVertexInstance vi in stage.Vertices)
{
if (vi.State == ExecutedVertexInstance.VertexState.Successful ||
vi.State == ExecutedVertexInstance.VertexState.Failed ||
(showCancelled && vi.State == ExecutedVertexInstance.VertexState.Cancelled))
{
StaticPartitionInformation spi = new StaticPartitionInformation(count++, vi.DataWritten, vi.Name + " v." + vi.Version);
result.AddPartition(spi);
if (vi.DataWritten != -1)
result.EstimatedSize += vi.DataWritten;
}
}
return result;
}
}
/// <summary>
/// User clicked on the job plan; figure out what to display.
/// </summary>
/// <param name="xcoord">X coordinate of click, in pixels.</param>
/// <param name="ycoord">Y coordinate of click, in pixels.</param>
/// <returns>If true, the event has been completely handled; else it will be handled by the default handler as well.</returns>
private void ClickOnJobPlan(int xcoord, int ycoord)
{
if (this.planVisible == PlanVisible.None)
return;
if (this.planVisible == PlanVisible.Static)
{
DryadJobStaticPlan.Stage stage;
object o = this.graphViewer.GetObjectAt(xcoord, ycoord);
var dEdge = o as DEdge;
if (dEdge != null)
{
Msagl.Drawing.Edge edge = (dEdge).DrawingEdge;
DryadJobStaticPlan.Connection connection = (DryadJobStaticPlan.Connection)edge.UserData;
StaticPartitionedTableInformation info =
StaticPartitionedTableInformation.StageOutput(this.Job, this.staticPlan, connection.From, this.Status, !this.hideCancelledVerticesToolStripMenuItem.Checked);
this.SetTable(info);
return;
}
var dNode = o as DNode;
if (dNode != null)
{
Msagl.Drawing.Node node = dNode.DrawingNode;
stage = (DryadJobStaticPlan.Stage)node.UserData;
}
else
{
this.SetNoStageOrTable("", false);
return;
}
if (stage.IsVirtual)
{
if (stage.IsInput ||
(this.Job.Summary.Status != ClusterJobInformation.ClusterJobStatus.Running && stage.IsOutput)) // can't display output tables if the job is not ready yet
{
// some input or output tables have associated processes; if that's true, display these instead of the table.
DryadLinqJobStage istage = this.Job.GetStage(stage.Name);
if (istage != null && istage.TotalInitiatedVertices != 0)
{
this.SetStage(istage);
}
else
{
this.Status("Reading table information", StatusKind.LongOp);
StaticPartitionedTableInformation info =
new StaticPartitionedTableInformation(this.Job.ClusterConfiguration, stage.UriType, stage.Uri, stage.Code, this.Status);
this.SetTable(info);
}
}
else if (stage.IsTee)
{
StaticPartitionedTableInformation info =
StaticPartitionedTableInformation.StageOutput(this.Job, this.staticPlan, stage, this.Status, !this.hideCancelledVerticesToolStripMenuItem.Checked);
this.SetTable(info);
return;
}
else
{
this.SetNoStageOrTable(stage.Name, stage.IsOutput);
}
return;
}
else
{
DryadLinqJobStage executedstage = this.Job.GetStage(stage.Name);
if (executedstage != null)
this.SetStage(executedstage);
}
}
else if (this.planVisible == PlanVisible.Dynamic)
{
// dynamic plan visible
double xo, yo;
this.planDrawSurface.ScreenToUser(xcoord, ycoord, out xo, out yo);
GraphLayout.GraphNode node = this.dynamicPlanLayout.FindNode(xo, yo);
this.dynamicPlanLayout.ClearSelectedNodes();
if (node != null)
{
node.Selected = true;
string stageName = node.Stage;
DryadLinqJobStage executedstage = this.Job.GetStage(stageName);
if (executedstage != null)
this.SetStage(executedstage);
}
else
{
this.SetNoStageOrTable("", false);
}
this.DrawQueryPlan(); // refresh the image
}
else if (this.planVisible == PlanVisible.Schedule)
{
double xo, yo;
this.planDrawSurface.ScreenToUser(xcoord, ycoord, out xo, out yo);
ExecutedVertexInstance vertex = this.jobSchedule.FindVertex(xo, yo);
this.SelectVertex(vertex);
}
return;
}
/// <summary>
/// Create a class representing the set of edges between two stages in the job plan.
/// These collectively look like a partitioned table.
/// If the source stage has multiple outputs there is not enough information to return meaningful information.
/// </summary>
/// <param name="job">Job whose slice we are displaying.</param>
/// <param name="plan">Static plan of the job.</param>
/// <param name="source">Stage in the job which produces the data.</param>
/// <param name="status">Delegate used to report errors.</param>
/// <param name="showCancelled">If true include cancelled vertices.</param>
public static StaticPartitionedTableInformation StageOutput(
DryadLinqJobInfo job,
DryadJobStaticPlan plan,
DryadJobStaticPlan.Stage source,
StatusReporter status,
bool showCancelled)
{
string header = "Output of " + source.Name;
// First check whether in the static plan this is virtual
while (source.IsTee)
{
var sourceInputs = plan.GetStageConnections(source, true).ToList();
if (sourceInputs.Count() != 1)
{
throw new DryadException("Unexpected number of inputs for stage " + source.Name);
}
source = sourceInputs.First().From;
}
// If we reached the input return information about that input
if (source.IsInput)
{
status("Scanning " + source.Name, StatusKind.LongOp);
StaticPartitionedTableInformation result = new StaticPartitionedTableInformation(job.ClusterConfiguration, source.UriType, source.Uri, source.Code, status);
result.Header = "Output of " + header;
result.constructorArguments = new SaveConstructorArguments
{
code = null,
source = source,
plan = plan
};
return(result);
}
else
{
StaticPartitionedTableInformation result = new StaticPartitionedTableInformation();
result.Name = "Output of vertices in stage " + source.Name;
result.Header = "Output of " + header;
result.constructorArguments = new SaveConstructorArguments
{
code = null,
source = source,
plan = plan
};
// Check whether this stage has multiple outputs; this can only happen for 'Fork' operators.
var destinations = plan.GetStageConnections(source, false);
if (destinations.Count() > 1)
{
result.Error = "Cannot provide information about one of multiple outputs of a stage.";
return(result);
}
DryadLinqJobStage stage = job.GetStage(source.Name);
if (stage == null)
{
result.Error = "There is no information about the output of stage " + source.Name;
return(result);
}
result.EstimatedSize = 0;
result.PartitionCount = stage.TotalInitiatedVertices;
int count = 0;
foreach (ExecutedVertexInstance vi in stage.Vertices)
{
if (vi.State == ExecutedVertexInstance.VertexState.Successful ||
vi.State == ExecutedVertexInstance.VertexState.Failed ||
(showCancelled && vi.State == ExecutedVertexInstance.VertexState.Cancelled))
{
StaticPartitionInformation spi = new StaticPartitionInformation(count++, vi.DataWritten, vi.Name + " v." + vi.Version);
result.AddPartition(spi);
if (vi.DataWritten != -1)
{
result.EstimatedSize += vi.DataWritten;
}
}
}
return(result);
}
}
