/// <summary>
/// Diagnose a list of jobs.
/// </summary>
/// <param name="jobs">Jobs to diagnose.</param>
/// <param name="config">Cluster configuration.</param>
/// <param name="manager">Communicatino manager.</param>
public static List <DiagnosisLog> DiagnoseJobs(IEnumerable <DryadLinqJobSummary> jobs, ClusterConfiguration config, CommManager manager)
{
var dryadLinqJobSummaries = jobs as DryadLinqJobSummary[] ?? jobs.ToArray();
int jobCount = dryadLinqJobSummaries.Count();
List <DiagnosisLog> result = new List <DiagnosisLog>();
int done = 0;
foreach (DryadLinqJobSummary summary in dryadLinqJobSummaries)
{
if (summary == null)
{
continue;
}
manager.Token.ThrowIfCancellationRequested();
JobFailureDiagnosis diagnosis = JobFailureDiagnosis.CreateJobFailureDiagnosis(config, summary, manager);
manager.Status("Diagnosing " + summary.ShortName(), StatusKind.LongOp);
DiagnosisLog log = diagnosis.Diagnose();
result.Add(log);
done++;
manager.Progress(done * 100 / jobCount);
}
manager.Status("Diagnosis complete", StatusKind.OK);
return(result);
}
/// <summary>
/// Recompute the list of jobs on the cluster and add them to the clusterJobs field.
/// </summary>
/// <param name="virtualCluster">Unused.</param>
/// <param name="manager">Communication manager.</param>
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary <string, ClusterJobInformation>();
if (string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
{
return;
}
string joblist = Path.Combine(CachedClusterResidentObject.CacheDirectory, "jobs");
if (!Directory.Exists(joblist))
{
Directory.CreateDirectory(joblist);
}
string[] files = Directory.GetFiles(joblist, "*.xml");
foreach (var file in files)
{
manager.Token.ThrowIfCancellationRequested();
DryadLinqJobSummary job = Utilities.LoadXml <DryadLinqJobSummary>(file);
string cjid = job.Cluster + "-" + job.ClusterJobId; // there may be two jobs with same id from different clusters
ClusterJobInformation ci = new ClusterJobInformation(this.Config.Name, job.Cluster, cjid, job.Name, job.User, job.Date, job.EndTime - job.Date, job.Status);
ci.SetAssociatedSummary(job);
if (this.clusterJobs.ContainsKey(cjid))
{
manager.Status("Duplicate job id, cannot insert in cache " + job.AsIdentifyingString(), StatusKind.Error);
continue;
}
this.clusterJobs.Add(cjid, ci);
}
manager.Progress(100);
}
/// <summary>
/// Try to find the job information from cluster and summary.
/// </summary>
/// <param name="manager">Communication manager.</param>
protected void FindJobInfo(CommManager manager)
{
DryadLinqJobInfo jobinfo = DryadLinqJobInfo.CreateDryadLinqJobInfo(this.cluster, this.Summary, true, manager);
if (jobinfo == null)
{
manager.Status("Cannot collect information for " + Summary.ShortName() + " to diagnose", StatusKind.Error);
return;
}
this.Job = jobinfo;
this.StaticPlan = JobObjectModel.DryadJobStaticPlan.CreatePlan(jobinfo, manager);
}
/// <summary>
/// Cancel a job.
/// </summary>
/// <param name="jobs">Jobs to cancel.</param>
/// <param name="cluster">Cluster where the jobs are running.</param>
/// <returns>True if all cancellations succeed.</returns>
/// <param name="manager">Communicatoni manager.</param>
// ReSharper disable once UnusedParameter.Global
public static bool CancelJobs(IEnumerable <DryadLinqJobSummary> jobs, ClusterStatus cluster, CommManager manager)
{
bool done = true;
foreach (DryadLinqJobSummary job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
if (job.Status != ClusterJobInformation.ClusterJobStatus.Running)
{
manager.Status("Job " + job.Name + " does not appear to be running; will still try to cancel", StatusKind.Error);
}
bool success;
string reason = "";
try
{
success = cluster.CancelJob(job);
}
catch (Exception ex)
{
success = false;
reason = ex.Message;
Trace.TraceInformation(ex.ToString());
}
if (success)
{
manager.Status("Job " + job.Name + " cancelled", StatusKind.OK);
}
else
{
manager.Status("Cancellation of " + job.Name + " failed " + reason, StatusKind.Error);
}
done &= success;
}
return(done);
}
/// <summary>
/// Diagnose a list of jobs.
/// </summary>
/// <param name="jobs">Jobs to diagnose.</param>
/// <param name="config">Cluster configuration.</param>
/// <param name="manager">Communicatino manager.</param>
public static List<DiagnosisLog> DiagnoseJobs(IEnumerable<DryadLinqJobSummary> jobs, ClusterConfiguration config, CommManager manager)
{
var dryadLinqJobSummaries = jobs as DryadLinqJobSummary[] ?? jobs.ToArray();
int jobCount = dryadLinqJobSummaries.Count();
List<DiagnosisLog> result = new List<DiagnosisLog>();
int done = 0;
foreach (DryadLinqJobSummary summary in dryadLinqJobSummaries)
{
if (summary == null) continue;
manager.Token.ThrowIfCancellationRequested();
JobFailureDiagnosis diagnosis = JobFailureDiagnosis.CreateJobFailureDiagnosis(config, summary, manager);
manager.Status("Diagnosing " + summary.ShortName(), StatusKind.LongOp);
DiagnosisLog log = diagnosis.Diagnose();
result.Add(log);
done++;
manager.Progress(done * 100 / jobCount);
}
manager.Status("Diagnosis complete", StatusKind.OK);
return result;
}
/// <summary>
/// Cancel a job.
/// </summary>
/// <param name="jobs">Jobs to cancel.</param>
/// <param name="cluster">Cluster where the jobs are running.</param>
/// <returns>True if all cancellations succeed.</returns>
/// <param name="manager">Communicatoni manager.</param>
// ReSharper disable once UnusedParameter.Global
public static bool CancelJobs(IEnumerable<DryadLinqJobSummary> jobs, ClusterStatus cluster, CommManager manager)
{
bool done = true;
foreach (DryadLinqJobSummary job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
if (job.Status != ClusterJobInformation.ClusterJobStatus.Running)
{
manager.Status("Job " + job.Name + " does not appear to be running; will still try to cancel", StatusKind.Error);
}
bool success;
string reason = "";
try
{
success = cluster.CancelJob(job);
}
catch (Exception ex)
{
success = false;
reason = ex.Message;
Trace.TraceInformation(ex.ToString());
}
if (success)
manager.Status("Job " + job.Name + " cancelled", StatusKind.OK);
else
manager.Status("Cancellation of " + job.Name + " failed " + reason, StatusKind.Error);
done &= success;
}
return done;
}
/// <summary>
/// Try to find the job information from cluster and summary.
/// </summary>
/// <param name="manager">Communication manager.</param>
protected void FindJobInfo(CommManager manager)
{
DryadLinqJobInfo jobinfo = DryadLinqJobInfo.CreateDryadLinqJobInfo(this.cluster, this.Summary, true, manager);
if (jobinfo == null)
{
manager.Status("Cannot collect information for " + Summary.ShortName() + " to diagnose", StatusKind.Error);
return;
}
this.Job = jobinfo;
this.StaticPlan = JobObjectModel.DryadJobStaticPlan.CreatePlan(jobinfo, manager);
}
/// <summary>
/// Recompute the list of jobs on the cluster and add them to the clusterJobs field.
/// </summary>
/// <param name="virtualCluster">Unused.</param>
/// <param name="manager">Communication manager.</param>
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
if (string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
return;
string joblist = Path.Combine(CachedClusterResidentObject.CacheDirectory, "jobs");
if (!Directory.Exists(joblist))
Directory.CreateDirectory(joblist);
string[] files = Directory.GetFiles(joblist, "*.xml");
foreach (var file in files)
{
manager.Token.ThrowIfCancellationRequested();
DryadLinqJobSummary job = Utilities.LoadXml<DryadLinqJobSummary>(file);
string cjid = job.Cluster + "-" + job.ClusterJobId; // there may be two jobs with same id from different clusters
ClusterJobInformation ci = new ClusterJobInformation(this.Config.Name, job.Cluster, cjid, job.Name, job.User, job.Date, job.EndTime - job.Date, job.Status);
ci.SetAssociatedSummary(job);
if (this.clusterJobs.ContainsKey(cjid))
{
manager.Status("Duplicate job id, cannot insert in cache " + job.AsIdentifyingString(), StatusKind.Error);
continue;
}
this.clusterJobs.Add(cjid, ci);
}
manager.Progress(100);
}
/// <summary>
/// Cache the vertices in the list; executed on the background thread.
/// </summary>
/// <returns>True: success.</returns>
/// <param name="manager">Communication manager.</param>
/// <param name="config">Cluster configuration.</param>
/// <param name="summary">Job to cache.</param>
/// <param name="vertices">Vertices to cache.</param>
private static bool CacheAllVertices(
ClusterConfiguration config, DryadLinqJobSummary summary, List<ExecutedVertexInstance> vertices,
CommManager manager)
{
int done = 0;
int todo = vertices.Count;
int files = 0;
manager.Status("Caching data for " + todo + " vertices", StatusKind.LongOp);
foreach (ExecutedVertexInstance v in vertices)
{
files += CacheVertexInfo(config, summary, v);
done++;
manager.Progress(done / todo);
}
manager.Progress(100);
manager.Status("Cached " + files + " files", StatusKind.OK);
return true;
}
/// <summary>
/// Get the contents of a specified cluster resident object.
/// </summary>
/// <param name="path">Cluster object whose contents is read.</param>
/// <param name="pattern">Pattern to filter contents, for folders.</param>
/// <returns>The file contents.</returns>
/// <param name="manager">Communication manager.</param>
private static FileContents GetContents(CommManager manager, IClusterResidentObject path, string pattern)
{
if (path == null)
{
return new FileContents("Null path");
}
StringBuilder output = new StringBuilder();
Dictionary<string, IClusterResidentObject> linkCache = new Dictionary<string, IClusterResidentObject>();
linkCache.Add(path.ToString(), path);
string error = (path.RepresentsAFolder ? "Folder " : "") + path;
if (path.Exception != null)
{
error += " [Error accessing: " + path.Exception.Message + "]";
return new FileContents(error);
}
if (path.RepresentsAFolder)
{
IEnumerable<IClusterResidentObject> dirs = path.GetFilesAndFolders(pattern);
int displayed = 0;
foreach (IClusterResidentObject d in dirs)
{
manager.Token.ThrowIfCancellationRequested();
if (d.Exception != null)
{
error += " [Error " + d.Exception.Message + "]";
return new FileContents(error);
}
if (d.RepresentsAFolder)
{
string dirdisplay = string.Format("{0:u} {1,16} file://{2}", d.CreationTime, "d", d.Name);
output.AppendLine(dirdisplay);
}
else
{
string filedisplay = string.Format("{0:u} {1,16:N0} file://{2}", d.CreationTime, d.Size, d.Name);
output.AppendLine(filedisplay);
}
linkCache.Add("file://" + d.Name, d);
displayed++;
}
if (displayed == 0)
error += "[empty]";
return new FileContents(output.ToString(), error, linkCache);
}
else
{
manager.Status("Extracting contents of " + path, StatusKind.LongOp);
ISharedStreamReader sr = path.GetStream();
if (sr.Exception != null)
{
error += " [Error " + sr.Exception.Message + "]";
return new FileContents(error);
}
else
{
if (path.Size == 0)
error += "[empty]";
var contents = sr.ReadToEnd(manager.Token);
return new FileContents(contents, error, linkCache);
}
}
}
/// <summary>
/// Fill the job info by parsing the stdout.txt.
/// <returns>The updated job.</returns>
/// <param name="manager">Communication manager.</param>
/// </summary>
public bool CollectEssentialInformation(CommManager manager)
{
this.RefreshJobStatus(manager);
if (this.ManagerVertex == null)
{
this.ManagerVertex = new ExecutedVertexInstance(this, -1, 0, "JobManager", "", this.Summary.Date);
this.ManagerVertex.IsManager = true;
this.ManagerVertex.SetStartInformation(this, this.Summary.Machine, this.Summary.Date, this.Summary.ManagerProcessGuid, "");
this.ManagerVertex.StartCommandTime = this.ManagerVertex.CreationTime = this.ManagerVertex.VertexScheduleTime = this.Summary.Date;
ExecutedVertexInstance.VertexState jmstate = ExecutedVertexInstance.VertexState.Started;
switch (this.Summary.Status)
{
case ClusterJobInformation.ClusterJobStatus.Failed:
jmstate = ExecutedVertexInstance.VertexState.Failed;
break;
/*
case ClusterJobInformation.ClusterJobStatus.Succeeded:
jmstate = ExecutedVertexInstance.VertexState.Successful;
break;
*/
}
this.ManagerVertex.SetState(jmstate);
this.jobVertices.Add(this.ManagerVertex);
}
if (this.stdoutpath == null)
return false;
bool success = this.ParseStdout(this.stdoutpath, manager);
manager.Progress(100);
if (!success)
return false;
this.JobInfoCannotBeCollected = false;
manager.Status("Stdout parsed", StatusKind.OK);
this.LastUpdatetime = DateTime.Now;
if (this.Summary.Status == ClusterJobInformation.ClusterJobStatus.Running)
{
foreach (var vertex in this.Vertices.Where(v => v.State == ExecutedVertexInstance.VertexState.Started))
vertex.MarkVertexWasRunning(this.LastUpdatetime);
this.ManagerVertex.MarkVertexWasRunning(this.LastUpdatetime);
}
else if (this.jobSummary.Status == ClusterJobInformation.ClusterJobStatus.Failed)
{
if (this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Started)
this.ManagerVertex.SetState(ExecutedVertexInstance.VertexState.Failed);
foreach (var vertex in this.Vertices.Where(v => v.State == ExecutedVertexInstance.VertexState.Started))
vertex.MarkVertexWasRunning(this.ManagerVertex.End);
}
return true;
}
/// <summary>
/// Create information about a job run on the cluster.
/// </summary>
/// <param name="cf">Cluster configuration.</param>
/// <param name="summary">Summary description of the job.</param>
/// <returns>The Dryad job description, or null.</returns>
/// <param name="fill">If true, fill all the information, otherwise the user will have to call FillInformation on the result later.</param>
/// <param name="manager">Communication manager.</param>
public static DryadLinqJobInfo CreateDryadLinqJobInfo(ClusterConfiguration cf, DryadLinqJobSummary summary, bool fill, CommManager manager)
{
try
{
DryadLinqJobInfo job = new DryadLinqJobInfo(cf, summary);
if (fill)
job.CollectEssentialInformation(manager);
return job;
}
catch (Exception e)
{
Trace.TraceInformation(e.ToString());
manager.Status("Could not collect job information for " + summary.Name + ": " + e.Message, StatusKind.Error);
return null;
}
}
/// <summary>
/// Discover the vertex channels in a Scope-generated vcmdStart*xml file.
/// </summary>
/// <returns>True if the discovery was successful.</returns>
/// <param name="inputs">If true discover the inputs.</param>
/// <param name="outputs">If true discover the outputs.</param>
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
/// <param name="manager">Communication manager.</param>
// ReSharper disable UnusedParameter.Global
public bool DiscoverScopeChannels(bool inputs, bool outputs, bool fast, CommManager manager)
// ReSharper restore UnusedParameter.Global
{
// find the xml file
var files = this.WorkDirectory.GetFilesAndFolders("vcmdStart*.xml").ToList();
if (files.Count != 1)
{
manager.Status("Cannot locate vcmdStart*.xml file", StatusKind.Error);
return false;
}
ISharedStreamReader sr = files.First().GetStream();
if (sr.Exception != null)
{
manager.Status("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error);
return false;
}
// ReSharper disable PossibleNullReferenceException
XDocument plan = XDocument.Parse(sr.ReadToEnd(manager.Token));
if (inputs && this.InputChannels == null)
{
var channels = new Dictionary<int, ChannelEndpointDescription>();
IEnumerable<XElement> inputsData = plan.Root.Element("inputs").Elements();
int chno = 0;
foreach (var e in inputsData)
{
string chpath = e.Attribute("path").Value;
long size = long.Parse(e.Attribute("length").Value);
ChannelEndpointDescription desc = new ChannelEndpointDescription(true, chno, chpath, size);
channels.Add(chno, desc);
chno++;
}
this.InputChannels = channels;
}
if (outputs && this.OutputChannels == null)
{
var channels = new Dictionary<int, ChannelEndpointDescription>();
IEnumerable<XElement> inputsData = plan.Root.Element("outputs").Elements();
int chno = 0;
foreach (var e in inputsData)
{
string chpath = e.Attribute("path").Value;
ChannelEndpointDescription desc = new ChannelEndpointDescription(true, chno, chpath, -1);
channels.Add(chno, desc);
chno ++;
}
this.OutputChannels = channels;
}
// ReSharper restore PossibleNullReferenceException
sr.Close();
return true;
}
/// <summary>
/// Factory: create the plan for a given job.
/// </summary>
/// <param name="dryadLinqJobInfo">Job to create plan for.</param>
/// <returns>The plan or null.</returns>
/// <param name="manager">Communication manager.</param>
public static DryadJobStaticPlan CreatePlan(DryadLinqJobInfo dryadLinqJobInfo, CommManager manager)
{
manager.Status("Trying to build static plan", StatusKind.LongOp);
ClusterConfiguration config = dryadLinqJobInfo.ClusterConfiguration;
IClusterResidentObject file = config.JobQueryPlan(dryadLinqJobInfo.Summary);
if (config is CacheClusterConfiguration)
config = (config as CacheClusterConfiguration).ActualConfig(dryadLinqJobInfo.Summary);
if (file.Exception == null)
{
DryadJobStaticPlan retval;
{
retval = new DryadLinqJobStaticPlan(config, file.GetStream());
}
retval.ParseQueryPlan(manager);
return retval;
}
else
{
manager.Status("Exception while looking for plan " + file.Exception.Message, StatusKind.Error);
return null;
}
}
/// <summary>
/// Parse the stdout.txt file from the job manager.
/// </summary>
/// <param name="file">File to parse.</param>
/// <param name="manager">Communication manager.</param>
/// <returns>True if the parsing succeeds.</returns>
private bool ParseStdout(IClusterResidentObject file, CommManager manager)
{
int currentLine = 0;
if (this.stdoutLinesParsed == 0)
// don't lose it if we are only parsing the tail.
this.lastTimestampSeen = this.Summary.Date; // start from the job submission timestamp
// we are reusing the stream
this.stdoutLinesParsed = 0;
try
{
long filesize = file.Size;
long readbytes = 0;
string message = "Scanning JM stdout " + file;
if (filesize >= 0)
message += string.Format("({0:N0} bytes)", filesize);
manager.Status(message, StatusKind.LongOp);
if (this.cachedStdoutReader == null)
this.cachedStdoutReader = file.GetStream();
if (this.cachedStdoutReader.Exception != null)
{
manager.Status("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error);
return false;
}
while (!this.cachedStdoutReader.EndOfStream)
{
string line = this.cachedStdoutReader.ReadLine();
readbytes += line.Length;
if (currentLine >= this.stdoutLinesParsed)
{
while (true)
{
manager.Token.ThrowIfCancellationRequested();
int startLine = currentLine;
bool completeLine = true;
try
{
completeLine = this.ParseStdoutLineNew(line);
}
catch (Exception ex)
{
manager.Status(string.Format("Line {0}: Exception {1}", currentLine, ex.Message), StatusKind.Error);
Console.WriteLine("Line {0}: Exception {1}", currentLine, ex);
}
if (!completeLine)
{
if (this.cachedStdoutReader.EndOfStream)
{
throw new Exception("File ended while scanning for closing quote started on line " + startLine);
}
string extraline = this.cachedStdoutReader.ReadLine();
line += "\n" + extraline;
currentLine++;
}
else break;
}
}
currentLine++;
if (currentLine % 100 == 0 && filesize > 0)
{
manager.Progress(Math.Min(100, (int)(100 * readbytes / filesize)));
}
}
this.stdoutLinesParsed = currentLine;
if (this.ManagerVertex != null)
{
if (this.ManagerVertex.End == DateTime.MinValue)
// approximation
this.ManagerVertex.End = this.lastTimestampSeen;
// we are done with this stream
if (this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Failed ||
this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Successful)
{
this.cachedStdoutReader.Close();
this.cachedStdoutReader = null; // will force reopening if refreshed
}
}
return true;
}
catch (Exception e)
{
manager.Status("Exception while reading stdout " + e.Message, StatusKind.Error);
Trace.TraceInformation(e.ToString());
return false;
}
}