/// <summary>
/// Extract the job information from a folder with logs on the local machine.
/// </summary>
/// <param name="jobRootFolder">Folder with logs for the specified job.</param>
/// <returns>The job information, or null if not found.</returns>
private ClusterJobInformation GetJobInfo(string jobRootFolder)
{
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, jobRootFolder);
long time;
long size;
this.config.DfsClient.GetFileStatus(uri, out time, out size);
DateTime date = DfsFile.TimeFromLong(time);
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
string jobName = Path.GetFileName(jobRootFolder);
string errorMsg = "";
try
{
var jobinfo = this.yarnClient.QueryJob(jobName, uri);
var jobstatus = jobinfo.GetStatus();
errorMsg = jobinfo.ErrorMsg;
switch (jobstatus)
{
case JobStatus.NotSubmitted:
case JobStatus.Waiting:
status = ClusterJobInformation.ClusterJobStatus.Unknown;
break;
case JobStatus.Running:
status = ClusterJobInformation.ClusterJobStatus.Running;
break;
case JobStatus.Success:
status = ClusterJobInformation.ClusterJobStatus.Succeeded;
break;
case JobStatus.Cancelled:
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
break;
case JobStatus.Failure:
status = ClusterJobInformation.ClusterJobStatus.Failed;
break;
default:
throw new ArgumentOutOfRangeException();
}
}
catch (Exception)
{
}
TimeSpan running = TimeSpan.Zero;
var info = new ClusterJobInformation(config.Name, "", jobName, jobName, Environment.UserName, date, running, status);
return(info);
}
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="conf">Configuration for the local machine.</param>
public HdfsClusterStatus(ClusterConfiguration conf)
: base(conf)
{
if (!(conf is HdfsClusterConfiguration))
{
throw new ArgumentException("Expected an HdfsClusterConfiguration, got a " + conf.GetType());
}
this.config = conf as HdfsClusterConfiguration;
// make a fake call to initialize the cluster on the foreground thread
// HDFS does not work if initialized on the background thread.
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
this.config.DfsClient.IsFileExists(uri); // ignore result
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
}
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
// ReSharper disable once UnusedParameter.Global
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary <string, ClusterJobInformation>();
var uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
var jobs = this.config.DfsClient.EnumerateSubdirectories(uri).ToList();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
ClusterJobInformation info = this.GetJobInfo(DfsFile.PathFromUri(this.config.JobsFolderUri, job));
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(info.ClusterJobID, info);
}
manager.Progress(100 * done++ / jobs.Count);
}
manager.Progress(100);
}