/// <summary> /// Recompute the list of jobs on the cluster and add them to the clusterJobs field. /// </summary> /// <param name="virtualCluster">Unused.</param> /// <param name="manager">Communication manager.</param> protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager) { this.clusterJobs = new Dictionary <string, ClusterJobInformation>(); if (string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory)) { return; } string joblist = Path.Combine(CachedClusterResidentObject.CacheDirectory, "jobs"); if (!Directory.Exists(joblist)) { Directory.CreateDirectory(joblist); } string[] files = Directory.GetFiles(joblist, "*.xml"); foreach (var file in files) { manager.Token.ThrowIfCancellationRequested(); DryadLinqJobSummary job = Utilities.LoadXml <DryadLinqJobSummary>(file); string cjid = job.Cluster + "-" + job.ClusterJobId; // there may be two jobs with same id from different clusters ClusterJobInformation ci = new ClusterJobInformation(this.Config.Name, job.Cluster, cjid, job.Name, job.User, job.Date, job.EndTime - job.Date, job.Status); ci.SetAssociatedSummary(job); if (this.clusterJobs.ContainsKey(cjid)) { manager.Status("Duplicate job id, cannot insert in cache " + job.AsIdentifyingString(), StatusKind.Error); continue; } this.clusterJobs.Add(cjid, ci); } manager.Progress(100); }
/// <summary> /// Initialize an empty cached cluster resident object. /// </summary> /// <param name="config">Cluster where the file resides.</param> /// <param name="job">Job who owns these files.</param> protected CachedClusterResidentObject(ClusterConfiguration config, DryadLinqJobSummary job) { this.cacheWriter = null; this.tempFileName = null; this.Job = job; this.Config = config; }
/// <summary> /// Not needed, all summaries are already known. /// </summary> /// <param name="job">Cluster job.</param> /// <returns>Throws an exception.</returns> /// <param name="manager">Communication manager.</param> public override ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, CommManager manager) { ClusterConfiguration actual = (this.Config as CacheClusterConfiguration).ActualConfig(job); ClusterStatus actualStatus = actual.CreateClusterStatus(); return(actualStatus.DiscoverClusterJob(job, manager)); }
// ReSharper restore UnusedParameter.Global /// <summary> /// Discover a cluster job given its id. /// </summary> /// <param name="job">Job to discover.</param> /// <returns>The cluster job, or null if not found.</returns> /// <param name="manager">Communication manager.</param> public virtual ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, CommManager manager) { if (this.clusterJobs == null) { this.RecomputeClusterJobList(job.VirtualCluster, manager); } return(this.clusterJobs[job.ClusterJobId]); }
/// <summary> /// Create a cluster resident object corresponding to a given pathname. /// </summary> /// <param name="path">Path to the cluster-resident object.</param> /// <param name="config">Cluster where the file resides.</param> /// <param name="shouldCache">If true the file should be cached.</param> /// <param name="job">Job who owns this file.</param> public UNCFile(ClusterConfiguration config, DryadLinqJobSummary job, UNCPathname path, bool shouldCache) : base(config, job) { this.Pathname = path; this.Exception = null; this.ShouldCacheLocally = shouldCache; //if (! this.RepresentsAFolder) this.LocalCachePath = this.CachePath(this.Pathname); }
/// <summary> /// Get all the files cached associated with a given job. /// </summary> /// <param name="job">Job with cached files.</param> /// <returns>An iterator over all files cached belonging to this job.</returns> public static IEnumerable <string> CachedJobFiles(DryadLinqJobSummary job) { if (perJobFiles.ContainsKey(job)) { return(perJobFiles[job]); } return(new List <string>()); }
/// <summary> /// Refresh the job summary status. /// </summary> /// <param name="summary">Summary to refresh.</param> /// <param name="manager">Communication manager.</param> public override void RefreshStatus(DryadLinqJobSummary summary, CommManager manager) { ClusterJobInformation info = this.GetJobInfo(summary.JobID); if (info == null) { summary.Status = ClusterJobInformation.ClusterJobStatus.Unknown; return; } summary.Status = info.Status; }
/// <summary> /// Refresh the job summary status. /// </summary> /// <param name="summary">Summary to refresh.</param> /// <param name="manager">Communication manager.</param> public virtual void RefreshStatus(DryadLinqJobSummary summary, CommManager manager) { // refresh the whole list: too expensive // this.RecomputeClusterJobList(summary.VirtualCluster, manager); ClusterJobInformation info = this.DiscoverClusterJob(summary, manager); if (info == null) { summary.Status = ClusterJobInformation.ClusterJobStatus.Unknown; return; } summary.Status = info.Status; }
/// <summary> /// A file with the specified path. /// </summary> /// <param name="path">Path to the file.</param> /// <param name="client">Azure client.</param> /// <param name="config">Cluster configuration.</param> /// <param name="job">Job accessing this file.</param> /// <param name="isFolder">If true this must be a folder.</param> /// <param name="canCache">True if the file can be cached (it is immutable for sure).</param> public AzureDfsFile(ClusterConfiguration config, DryadLinqJobSummary job, AzureDfsClient client, string path, bool canCache, bool isFolder) : base(config, job) { this.client = client; this.path = path; this.ShouldCacheLocally = canCache; this.RepresentsAFolder = isFolder; this.size = -1; if (!string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory)) { this.LocalCachePath = Path.Combine(CachedClusterResidentObject.CacheDirectory, this.path); } }
/// <summary> /// Record that the job owns this cached file. /// </summary> /// <param name="job">Job.</param> /// <param name="path">Cached file belonging to this job.</param> public static void RecordCachedFile(DryadLinqJobSummary job, string path) { HashSet <string> list; if (!perJobFiles.ContainsKey(job)) { list = new HashSet <string>(); perJobFiles.Add(job, list); } else { list = perJobFiles[job]; } list.Add(path); }
/// <summary> /// Refresh the job summary status. /// </summary> /// <param name="job">Summary to refresh.</param> /// <param name="manager">Communication manager.</param> public override void RefreshStatus(DryadLinqJobSummary job, CommManager manager) { ClusterConfiguration actual = (this.Config as CacheClusterConfiguration).ActualConfig(job); ClusterStatus actualStatus = actual.CreateClusterStatus(); actualStatus.RefreshStatus(job, manager); ClusterJobInformation info = actualStatus.DiscoverClusterJob(job, manager); if (info == null) { job.Status = ClusterJobInformation.ClusterJobStatus.Unknown; return; } job.Status = info.Status; }
/// <summary> /// Discover the (unique) dryadlinq job corresponding to a cluster job. /// </summary> /// <param name="clusterJob">Cluster Job.</param> /// <returns>The job description.</returns> /// <param name="reporter">Delegate used to report errors.</param> public override DryadLinqJobSummary DiscoverDryadLinqJobFromClusterJob(ClusterJobInformation clusterJob, StatusReporter reporter) { DryadLinqJobSummary result = new DryadLinqJobSummary( clusterJob.Cluster, this.Config.TypeOfCluster, "", // virtual cluster "", // machine clusterJob.ClusterJobID, // jobId clusterJob.ClusterJobID, // clusterJobId new DryadProcessIdentifier("jm"), // jmProcessGuid clusterJob.Name, clusterJob.User, clusterJob.Date, clusterJob.Date + clusterJob.EstimatedRunningTime, clusterJob.Status); return(result); }
/// <summary> /// Discover the dryadlinq job associated with a cluster job. /// </summary> /// <param name="status">Cluster configuration.</param> /// <returns>The job, if any</returns> /// <param name="reporter">Delegate used to report errors.</param> public DryadLinqJobSummary DiscoverDryadLinqJob(ClusterStatus status, StatusReporter reporter) { if (this.IsUnavailable) { return(null); } if (this.JobSummary != null) { return(this.JobSummary); } DryadLinqJobSummary j = status.DiscoverDryadLinqJobFromClusterJob(this, reporter); if (j == null) { this.IsUnavailable = true; } return(this.JobSummary = j); }
/// <summary> /// Cancel the specified job. /// </summary> /// <param name="job">Job whose execution is cancelled.</param> /// <returns>True if the cancellation succeeded.</returns> public override bool CancelJob(DryadLinqJobSummary job) { Microsoft.Research.Peloponnese.Azure.Utils.KillJob(this.config.AccountName, this.config.AccountKey, this.config.Container, job.ClusterJobId); return(false); }
/// <summary> /// Cancel the specified job. /// </summary> /// <param name="job">Job whose execution is cancelled.</param> /// <returns>True if the cancellation succeeded.</returns> public override bool CancelJob(DryadLinqJobSummary job) { return(false); }
/// <summary> /// If known, set the associated job summary. /// </summary> /// <param name="summary">Job summary for this cluster job.</param> public void SetAssociatedSummary(DryadLinqJobSummary summary) { this.JobSummary = summary; }
/// <summary> /// Cancel the specified job. /// </summary> /// <param name="job">Job whose execution is cancelled.</param> /// <returns>True if the cancellation succeeded.</returns> public abstract bool CancelJob(DryadLinqJobSummary job);
/// <summary> /// Cancel the specified job. /// </summary> /// <param name="job">Job whose execution is cancelled.</param> /// <returns>True if the cancellation succeeded.</returns> public override bool CancelJob(DryadLinqJobSummary job) { throw new InvalidOperationException(); }
/// <summary> /// Cancel the specified job. /// </summary> /// <param name="job">Job whose execution is cancelled.</param> /// <returns>True if the cancellation succeeded.</returns> public override bool CancelJob(DryadLinqJobSummary job) { AzureUtils.KillJob(this.config.AccountName, this.config.AccountKey, this.config.Container, job.ClusterJobId); return(true); }