private void filteredDataGridView_CellFormatting(object sender, DataGridViewCellFormattingEventArgs e)
{
ClusterJobInformation t = (ClusterJobInformation)this.filteredDataGridView.DataGridView.Rows[e.RowIndex].DataBoundItem;
switch (t.Status)
{
case ClusterJobInformation.ClusterJobStatus.Cancelled:
e.CellStyle.BackColor = Color.Yellow;
break;
case ClusterJobInformation.ClusterJobStatus.Succeeded:
e.CellStyle.BackColor = Color.LightGreen;
break;
case ClusterJobInformation.ClusterJobStatus.Failed:
e.CellStyle.BackColor = Color.Tomato;
break;
case ClusterJobInformation.ClusterJobStatus.Unknown:
e.CellStyle.BackColor = Color.White;
break;
case ClusterJobInformation.ClusterJobStatus.Running:
e.CellStyle.BackColor = Color.Cyan;
break;
}
if (t.IsUnavailable)
{
e.CellStyle.BackColor = Color.Gray;
}
}
/// <summary>
/// Find cluster jobs selected in the upper pane.
/// </summary>
/// <returns>A reference to the cluster job information.</returns>
private IEnumerable <ClusterJobInformation> SelectedJobs()
{
DataGridViewSelectedRowCollection sel = this.filteredDataGridView.DataGridView.SelectedRows;
if (sel.Count < 1)
{
Status("You must select exactly some rows in the upper pane.", StatusKind.Error);
yield break;
}
for (int i = 0; i < sel.Count; i++)
{
ClusterJobInformation ti = sel[i].DataBoundItem as ClusterJobInformation;
yield return(ti);
}
}
/// <summary>
/// The user has double-clicked a row in the clusterJobTable. Browse the job.
/// </summary>
/// <param name="sender">Unused.</param>
/// <param name="e">Event describing position.</param>
private void filteredDataGridView_CellMouseDoubleClick(object sender, DataGridViewCellMouseEventArgs e)
{
if (e.RowIndex < 0)
{
return;
}
ClusterJobInformation task = (ClusterJobInformation)this.filteredDataGridView.DataGridView.Rows[e.RowIndex].DataBoundItem;
DryadLinqJobSummary js = task.DiscoverDryadLinqJob(this.clusterStatus, this.Status);
if (js == null)
{
this.Status("Error discovering job information on cluster.", StatusKind.Error);
}
else
{
this.Status("Starting job browser...", StatusKind.LongOp);
this.browseFromJobSummary(js);
}
}
/// <summary>
/// Extract the job information from a folder with logs on the local machine.
/// </summary>
/// <param name="jobRootFolder">Folder with logs for the specified job.</param>
/// <returns>The job information, or null if not found.</returns>
private ClusterJobInformation GetJobInfo(string jobRootFolder)
{
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, jobRootFolder);
long time;
long size;
this.config.DfsClient.GetFileStatus(uri, out time, out size);
DateTime date = DfsFile.TimeFromLong(time);
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
string jobName = Path.GetFileName(jobRootFolder);
string errorMsg = "";
try
{
var jobinfo = this.yarnClient.QueryJob(jobName, uri);
var jobstatus = jobinfo.GetStatus();
errorMsg = jobinfo.ErrorMsg;
switch (jobstatus)
{
case JobStatus.NotSubmitted:
case JobStatus.Waiting:
status = ClusterJobInformation.ClusterJobStatus.Unknown;
break;
case JobStatus.Running:
status = ClusterJobInformation.ClusterJobStatus.Running;
break;
case JobStatus.Success:
status = ClusterJobInformation.ClusterJobStatus.Succeeded;
break;
case JobStatus.Cancelled:
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
break;
case JobStatus.Failure:
status = ClusterJobInformation.ClusterJobStatus.Failed;
break;
default:
throw new ArgumentOutOfRangeException();
}
}
catch (Exception)
{
}
TimeSpan running = TimeSpan.Zero;
var info = new ClusterJobInformation(config.Name, "", jobName, jobName, Environment.UserName, date, running, status);
return info;
}
/// <summary>
/// Discover the (unique) dryadlinq job corresponding to a cluster job.
/// </summary>
/// <param name="clusterJob">Cluster Job.</param>
/// <returns>The job description.</returns>
/// <param name="reporter">Delegate used to report errors.</param>
public override DryadLinqJobSummary DiscoverDryadLinqJobFromClusterJob(ClusterJobInformation clusterJob, StatusReporter reporter)
{
DryadLinqJobSummary result = new DryadLinqJobSummary(
clusterJob.Cluster,
this.Config.TypeOfCluster,
"", // virtual cluster
"", // machine
clusterJob.ClusterJobID, // jobId
clusterJob.ClusterJobID, // clusterJobId
new DryadProcessIdentifier("jm"), // jmProcessGuid
clusterJob.Name,
clusterJob.User,
clusterJob.Date,
clusterJob.Date + clusterJob.EstimatedRunningTime,
clusterJob.Status);
return result;
}
/// <summary>
/// Extract the job information from a folder with logs on the local machine.
/// </summary>
/// <param name="jobRootFolder">Folder with logs for the specified job.</param>
/// <returns>The job information, or null if not found.</returns>
private ClusterJobInformation GetJobInfo(string jobRootFolder)
{
DateTime date = DateTime.MinValue;
DateTime lastHeartBeat = DateTime.MinValue;
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
bool found = false;
Uri uri = AzureDfsFile.UriFromPath(this.config, jobRootFolder);
var jobsFolders = this.config.AzureClient.ExpandFileOrDirectory(uri).ToList();
jobRootFolder = GetBlobName(this.config.Container, jobRootFolder);
string jobName = jobRootFolder;
foreach (var file in jobsFolders)
{
if (file.AbsolutePath.EndsWith("heartbeat"))
{
string blobName = GetBlobName(this.config.Container, file.AbsolutePath);
var blob = this.config.AzureClient.Container.GetPageBlobReference(blobName);
blob.FetchAttributes();
var props = blob.Metadata;
if (props.ContainsKey("status"))
{
var st = props["status"];
switch (st)
{
case "failure":
status = ClusterJobInformation.ClusterJobStatus.Failed;
break;
case "success":
status = ClusterJobInformation.ClusterJobStatus.Succeeded;
break;
case "running":
status = ClusterJobInformation.ClusterJobStatus.Running;
break;
case "killed":
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
break;
default:
Console.WriteLine("Unknown status " + st);
break;
}
}
if (props.ContainsKey("heartbeat"))
{
var hb = props["heartbeat"];
if (DateTime.TryParse(hb, out lastHeartBeat))
{
lastHeartBeat = lastHeartBeat.ToLocalTime();
if (status == ClusterJobInformation.ClusterJobStatus.Running &&
DateTime.Now - lastHeartBeat > TimeSpan.FromSeconds(40))
// job has in fact crashed
status = ClusterJobInformation.ClusterJobStatus.Failed;
}
}
if (props.ContainsKey("jobname"))
{
jobName = props["jobname"];
}
if (props.ContainsKey("starttime"))
{
var t = props["starttime"];
if (DateTime.TryParse(t, out date))
date = date.ToLocalTime();
}
found = true;
}
else if (file.AbsolutePath.Contains("DryadLinqProgram__") &&
// newer heartbeats contain the date
date != DateTime.MinValue)
{
var blob = this.config.AzureClient.Container.GetBlockBlobReference(AzureDfsFile.PathFromUri(this.config, file));
blob.FetchAttributes();
var props = blob.Properties;
if (props.LastModified.HasValue)
{
date = props.LastModified.Value.DateTime;
date = date.ToLocalTime();
}
}
}
if (!found) return null;
TimeSpan running = TimeSpan.Zero;
if (date != DateTime.MinValue && lastHeartBeat != DateTime.MinValue)
running = lastHeartBeat - date;
var info = new ClusterJobInformation(this.config.Name, "", jobRootFolder, jobName, Environment.UserName, date, running, status);
return info;
}
/// <summary>
/// Extract the job information from a folder with logs on the local machine.
/// </summary>
/// <param name="jobRootFolder">Folder with logs for the specified job.</param>
/// <returns>The job information, or null if not found.</returns>
/// <param name="jobId">Job id.</param>
private ClusterJobInformation GetJobInfo(string jobRootFolder, string jobId)
{
string jmFolder = Path.Combine(jobRootFolder, "jm");
if (!Directory.Exists(jmFolder)) return null;
var date = File.GetCreationTime(jmFolder);
ClusterJobInformation info = new ClusterJobInformation(this.config.Name, "", jobId, jobId, Environment.UserName, date, TimeSpan.Zero, ClusterJobInformation.ClusterJobStatus.Unknown);
return info;
}
/// <summary>
/// Not needed, all summaries are already known.
/// </summary>
/// <param name="clusterJob">Cluster job information.</param>
/// <param name="reporter">Delegate used to report errors.</param>
/// <returns>Throws an exception.</returns>
public override DryadLinqJobSummary DiscoverDryadLinqJobFromClusterJob(ClusterJobInformation clusterJob, StatusReporter reporter)
{
throw new InvalidOperationException();
}
/// <summary>
/// Recompute the list of jobs on the cluster and add them to the clusterJobs field.
/// </summary>
/// <param name="virtualCluster">Unused.</param>
/// <param name="manager">Communication manager.</param>
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
if (string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
return;
string joblist = Path.Combine(CachedClusterResidentObject.CacheDirectory, "jobs");
if (!Directory.Exists(joblist))
Directory.CreateDirectory(joblist);
string[] files = Directory.GetFiles(joblist, "*.xml");
foreach (var file in files)
{
manager.Token.ThrowIfCancellationRequested();
DryadLinqJobSummary job = Utilities.LoadXml<DryadLinqJobSummary>(file);
string cjid = job.Cluster + "-" + job.ClusterJobId; // there may be two jobs with same id from different clusters
ClusterJobInformation ci = new ClusterJobInformation(this.Config.Name, job.Cluster, cjid, job.Name, job.User, job.Date, job.EndTime - job.Date, job.Status);
ci.SetAssociatedSummary(job);
if (this.clusterJobs.ContainsKey(cjid))
{
manager.Status("Duplicate job id, cannot insert in cache " + job.AsIdentifyingString(), StatusKind.Error);
continue;
}
this.clusterJobs.Add(cjid, ci);
}
manager.Progress(100);
}
/// <summary> /// Discover the (unique) dryadlinq job corresponding to a cluster job. /// </summary> /// <param name="clusterJob">Cluster Job.</param> /// <returns>The job description.</returns> /// <param name="reporter">Delegate used to report errors.</param> public abstract DryadLinqJobSummary DiscoverDryadLinqJobFromClusterJob(ClusterJobInformation clusterJob, StatusReporter reporter);
/// <summary>
/// Decide whether the job has finished executing.
/// </summary>
/// <returns>A decision.</returns>
protected Decision IsJobFinished()
{
bool dec = ClusterJobInformation.JobIsFinished(this.Summary.Status);
return(dec ? Decision.Yes : Decision.No);
}
