/// <summary>
/// Check whether this vertex is reading from a job input.
/// </summary>
/// <returns>The list of input stages this vertex is reading from.</returns>
protected IEnumerable <DryadJobStaticPlan.Stage> VertexIsReadingFromJobInput()
{
if (this.StaticPlan == null)
{
yield break;
}
string stage = this.Vertex.StageName;
DryadJobStaticPlan.Stage staticStage = this.StaticPlan.GetStageByName(stage);
if (staticStage == null)
{
yield break;
}
foreach (DryadJobStaticPlan.Connection connection in this.StaticPlan.GetStageConnections(staticStage, true))
{
var input = connection.From;
if (input.IsInput)
{
yield return(input);
}
}
}
/// <summary>
/// Create a class representing the set of edges between two stages in the job plan.
/// These collectively look like a partitioned table.
/// If the source stage has multiple outputs there is not enough information to return meaningful information.
/// </summary>
/// <param name="job">Job whose slice we are displaying.</param>
/// <param name="plan">Static plan of the job.</param>
/// <param name="source">Stage in the job which produces the data.</param>
/// <param name="status">Delegate used to report errors.</param>
/// <param name="showCancelled">If true include cancelled vertices.</param>
public static StaticPartitionedTableInformation StageOutput(
DryadLinqJobInfo job,
DryadJobStaticPlan plan,
DryadJobStaticPlan.Stage source,
StatusReporter status,
bool showCancelled)
{
string header = "Output of " + source.Name;
// First check whether in the static plan this is virtual
while (source.IsTee)
{
var sourceInputs = plan.GetStageConnections(source, true).ToList();
if (sourceInputs.Count() != 1)
{
throw new DryadException("Unexpected number of inputs for stage " + source.Name);
}
source = sourceInputs.First().From;
}
// If we reached the input return information about that input
if (source.IsInput)
{
status("Scanning " + source.Name, StatusKind.LongOp);
StaticPartitionedTableInformation result = new StaticPartitionedTableInformation(job.ClusterConfiguration, source.UriType, source.Uri, source.Code, status);
result.Header = "Output of " + header;
result.constructorArguments = new SaveConstructorArguments
{
code = null,
source = source,
plan = plan
};
return(result);
}
else
{
StaticPartitionedTableInformation result = new StaticPartitionedTableInformation();
result.Name = "Output of vertices in stage " + source.Name;
result.Header = "Output of " + header;
result.constructorArguments = new SaveConstructorArguments
{
code = null,
source = source,
plan = plan
};
// Check whether this stage has multiple outputs; this can only happen for 'Fork' operators.
var destinations = plan.GetStageConnections(source, false);
if (destinations.Count() > 1)
{
result.Error = "Cannot provide information about one of multiple outputs of a stage.";
return(result);
}
DryadLinqJobStage stage = job.GetStage(source.Name);
if (stage == null)
{
result.Error = "There is no information about the output of stage " + source.Name;
return(result);
}
result.EstimatedSize = 0;
result.PartitionCount = stage.TotalInitiatedVertices;
int count = 0;
foreach (ExecutedVertexInstance vi in stage.Vertices)
{
if (vi.State == ExecutedVertexInstance.VertexState.Successful ||
vi.State == ExecutedVertexInstance.VertexState.Failed ||
(showCancelled && vi.State == ExecutedVertexInstance.VertexState.Cancelled))
{
StaticPartitionInformation spi = new StaticPartitionInformation(count++, vi.DataWritten, vi.Name + " v." + vi.Version);
result.AddPartition(spi);
if (vi.DataWritten != -1)
{
result.EstimatedSize += vi.DataWritten;
}
}
}
return(result);
}
}