/// <summary>
/// Recompute the list of jobs on the cluster and add them to the clusterJobs field.
/// </summary>
/// <param name="virtualCluster">Unused.</param>
/// <param name="manager">Communication manager.</param>
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary <string, ClusterJobInformation>();
if (string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
{
return;
}
string joblist = Path.Combine(CachedClusterResidentObject.CacheDirectory, "jobs");
if (!Directory.Exists(joblist))
{
Directory.CreateDirectory(joblist);
}
string[] files = Directory.GetFiles(joblist, "*.xml");
foreach (var file in files)
{
manager.Token.ThrowIfCancellationRequested();
DryadLinqJobSummary job = Utilities.LoadXml <DryadLinqJobSummary>(file);
string cjid = job.Cluster + "-" + job.ClusterJobId; // there may be two jobs with same id from different clusters
ClusterJobInformation ci = new ClusterJobInformation(this.Config.Name, job.Cluster, cjid, job.Name, job.User, job.Date, job.EndTime - job.Date, job.Status);
ci.SetAssociatedSummary(job);
if (this.clusterJobs.ContainsKey(cjid))
{
manager.Status("Duplicate job id, cannot insert in cache " + job.AsIdentifyingString(), StatusKind.Error);
continue;
}
this.clusterJobs.Add(cjid, ci);
}
manager.Progress(100);
}
/// <summary>
/// Initialize an empty cached cluster resident object.
/// </summary>
/// <param name="config">Cluster where the file resides.</param>
/// <param name="job">Job who owns these files.</param>
protected CachedClusterResidentObject(ClusterConfiguration config, DryadLinqJobSummary job)
{
this.cacheWriter = null;
this.tempFileName = null;
this.Job = job;
this.Config = config;
}
/// <summary>
/// Not needed, all summaries are already known.
/// </summary>
/// <param name="job">Cluster job.</param>
/// <returns>Throws an exception.</returns>
/// <param name="manager">Communication manager.</param>
public override ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, CommManager manager)
{
ClusterConfiguration actual = (this.Config as CacheClusterConfiguration).ActualConfig(job);
ClusterStatus actualStatus = actual.CreateClusterStatus();
return(actualStatus.DiscoverClusterJob(job, manager));
}
// ReSharper restore UnusedParameter.Global
/// <summary>
/// Discover a cluster job given its id.
/// </summary>
/// <param name="job">Job to discover.</param>
/// <returns>The cluster job, or null if not found.</returns>
/// <param name="manager">Communication manager.</param>
public virtual ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, CommManager manager)
{
if (this.clusterJobs == null)
{
this.RecomputeClusterJobList(job.VirtualCluster, manager);
}
return(this.clusterJobs[job.ClusterJobId]);
}
/// <summary>
/// Create a cluster resident object corresponding to a given pathname.
/// </summary>
/// <param name="path">Path to the cluster-resident object.</param>
/// <param name="config">Cluster where the file resides.</param>
/// <param name="shouldCache">If true the file should be cached.</param>
/// <param name="job">Job who owns this file.</param>
public UNCFile(ClusterConfiguration config, DryadLinqJobSummary job, UNCPathname path, bool shouldCache) : base(config, job)
{
this.Pathname = path;
this.Exception = null;
this.ShouldCacheLocally = shouldCache;
//if (! this.RepresentsAFolder)
this.LocalCachePath = this.CachePath(this.Pathname);
}
/// <summary>
/// Get all the files cached associated with a given job.
/// </summary>
/// <param name="job">Job with cached files.</param>
/// <returns>An iterator over all files cached belonging to this job.</returns>
public static IEnumerable <string> CachedJobFiles(DryadLinqJobSummary job)
{
if (perJobFiles.ContainsKey(job))
{
return(perJobFiles[job]);
}
return(new List <string>());
}
/// <summary>
/// Refresh the job summary status.
/// </summary>
/// <param name="summary">Summary to refresh.</param>
/// <param name="manager">Communication manager.</param>
public override void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
{
ClusterJobInformation info = this.GetJobInfo(summary.JobID);
if (info == null)
{
summary.Status = ClusterJobInformation.ClusterJobStatus.Unknown;
return;
}
summary.Status = info.Status;
}
/// <summary>
/// Refresh the job summary status.
/// </summary>
/// <param name="summary">Summary to refresh.</param>
/// <param name="manager">Communication manager.</param>
public virtual void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
{
// refresh the whole list: too expensive
// this.RecomputeClusterJobList(summary.VirtualCluster, manager);
ClusterJobInformation info = this.DiscoverClusterJob(summary, manager);
if (info == null)
{
summary.Status = ClusterJobInformation.ClusterJobStatus.Unknown;
return;
}
summary.Status = info.Status;
}
/// <summary>
/// A file with the specified path.
/// </summary>
/// <param name="path">Path to the file.</param>
/// <param name="client">Azure client.</param>
/// <param name="config">Cluster configuration.</param>
/// <param name="job">Job accessing this file.</param>
/// <param name="isFolder">If true this must be a folder.</param>
/// <param name="canCache">True if the file can be cached (it is immutable for sure).</param>
public AzureDfsFile(ClusterConfiguration config, DryadLinqJobSummary job, AzureDfsClient client, string path, bool canCache, bool isFolder)
: base(config, job)
{
this.client = client;
this.path = path;
this.ShouldCacheLocally = canCache;
this.RepresentsAFolder = isFolder;
this.size = -1;
if (!string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
{
this.LocalCachePath = Path.Combine(CachedClusterResidentObject.CacheDirectory, this.path);
}
}
/// <summary>
/// Record that the job owns this cached file.
/// </summary>
/// <param name="job">Job.</param>
/// <param name="path">Cached file belonging to this job.</param>
public static void RecordCachedFile(DryadLinqJobSummary job, string path)
{
HashSet <string> list;
if (!perJobFiles.ContainsKey(job))
{
list = new HashSet <string>();
perJobFiles.Add(job, list);
}
else
{
list = perJobFiles[job];
}
list.Add(path);
}
/// <summary>
/// Refresh the job summary status.
/// </summary>
/// <param name="job">Summary to refresh.</param>
/// <param name="manager">Communication manager.</param>
public override void RefreshStatus(DryadLinqJobSummary job, CommManager manager)
{
ClusterConfiguration actual = (this.Config as CacheClusterConfiguration).ActualConfig(job);
ClusterStatus actualStatus = actual.CreateClusterStatus();
actualStatus.RefreshStatus(job, manager);
ClusterJobInformation info = actualStatus.DiscoverClusterJob(job, manager);
if (info == null)
{
job.Status = ClusterJobInformation.ClusterJobStatus.Unknown;
return;
}
job.Status = info.Status;
}
/// <summary>
/// Discover the (unique) dryadlinq job corresponding to a cluster job.
/// </summary>
/// <param name="clusterJob">Cluster Job.</param>
/// <returns>The job description.</returns>
/// <param name="reporter">Delegate used to report errors.</param>
public override DryadLinqJobSummary DiscoverDryadLinqJobFromClusterJob(ClusterJobInformation clusterJob, StatusReporter reporter)
{
DryadLinqJobSummary result = new DryadLinqJobSummary(
clusterJob.Cluster,
this.Config.TypeOfCluster,
"", // virtual cluster
"", // machine
clusterJob.ClusterJobID, // jobId
clusterJob.ClusterJobID, // clusterJobId
new DryadProcessIdentifier("jm"), // jmProcessGuid
clusterJob.Name,
clusterJob.User,
clusterJob.Date,
clusterJob.Date + clusterJob.EstimatedRunningTime,
clusterJob.Status);
return(result);
}
/// <summary>
/// Discover the dryadlinq job associated with a cluster job.
/// </summary>
/// <param name="status">Cluster configuration.</param>
/// <returns>The job, if any</returns>
/// <param name="reporter">Delegate used to report errors.</param>
public DryadLinqJobSummary DiscoverDryadLinqJob(ClusterStatus status, StatusReporter reporter)
{
if (this.IsUnavailable)
{
return(null);
}
if (this.JobSummary != null)
{
return(this.JobSummary);
}
DryadLinqJobSummary j = status.DiscoverDryadLinqJobFromClusterJob(this, reporter);
if (j == null)
{
this.IsUnavailable = true;
}
return(this.JobSummary = j);
}
/// <summary>
/// Cancel the specified job.
/// </summary>
/// <param name="job">Job whose execution is cancelled.</param>
/// <returns>True if the cancellation succeeded.</returns>
public override bool CancelJob(DryadLinqJobSummary job)
{
Microsoft.Research.Peloponnese.Azure.Utils.KillJob(this.config.AccountName, this.config.AccountKey, this.config.Container, job.ClusterJobId);
return(false);
}
/// <summary>
/// Cancel the specified job.
/// </summary>
/// <param name="job">Job whose execution is cancelled.</param>
/// <returns>True if the cancellation succeeded.</returns>
public override bool CancelJob(DryadLinqJobSummary job)
{
return(false);
}
/// <summary>
/// If known, set the associated job summary.
/// </summary>
/// <param name="summary">Job summary for this cluster job.</param>
public void SetAssociatedSummary(DryadLinqJobSummary summary)
{
this.JobSummary = summary;
}
/// <summary> /// Cancel the specified job. /// </summary> /// <param name="job">Job whose execution is cancelled.</param> /// <returns>True if the cancellation succeeded.</returns> public abstract bool CancelJob(DryadLinqJobSummary job);
/// <summary>
/// Cancel the specified job.
/// </summary>
/// <param name="job">Job whose execution is cancelled.</param>
/// <returns>True if the cancellation succeeded.</returns>
public override bool CancelJob(DryadLinqJobSummary job)
{
throw new InvalidOperationException();
}
/// <summary>
/// Cancel the specified job.
/// </summary>
/// <param name="job">Job whose execution is cancelled.</param>
/// <returns>True if the cancellation succeeded.</returns>
public override bool CancelJob(DryadLinqJobSummary job)
{
AzureUtils.KillJob(this.config.AccountName, this.config.AccountKey, this.config.Container, job.ClusterJobId);
return(true);
}