/// <summary>
/// Try to diagnose whether there's a CLR mismatch error.
/// </summary>
/// <returns>True if this is the problem.</returns>
protected bool CLRStartupProblems()
{
IClusterResidentObject stdout = this.Job.ClusterConfiguration.ProcessStdoutFile(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary);
if (stdout.Exception != null)
{
return(false);
}
ISharedStreamReader sr = stdout.GetStream();
// only look for the error in the first 10 lines
for (int i = 0; i < 10; i++)
{
if (sr.EndOfStream)
{
sr.Close();
return(false);
}
string line = sr.ReadLine();
if (line.Contains("Error code 2148734720 (0x80131700)"))
{
this.Log(DiagnosisMessage.Importance.Final, "Error found in vertex stdout:", line);
sr.Close();
return(true);
}
}
sr.Close();
return(false);
}
/// <summary>
/// The stack trace of the vertex at the time of the crash.
/// </summary>
/// <returns>The stack trace or an empty collection.</returns>
public virtual IEnumerable <string> StackTrace()
{
IClusterResidentObject logdir = this.Job.ClusterConfiguration.ProcessWorkDirectory(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary);
IClusterResidentObject stackTrace = logdir.GetFile(this.stackTraceFile);
ISharedStreamReader sr = stackTrace.GetStream();
if (sr.Exception == null)
{
while (!sr.EndOfStream)
{
yield return(sr.ReadLine());
}
}
else
{
yield break;
}
}
/// <summary>
/// Discover whether the failure is caused by the inability to parse the XML plan.
/// </summary>
/// <returns>The decision.</returns>
public Decision XmlPlanParseError()
{
if (this.jobManager == null)
{
this.Log(DiagnosisMessage.Importance.Tracing, "Could not find job manager vertex information", "");
return(Decision.Dontknow);
}
IClusterResidentObject jmstdout = this.jobManager.StdoutFile;
if (jmstdout.Exception != null)
{
this.Log(DiagnosisMessage.Importance.Tracing, "Could not find job manager standard output", "");
return(Decision.Dontknow);
}
ISharedStreamReader sr = jmstdout.GetStream();
if (sr.Exception != null)
{
this.Log(DiagnosisMessage.Importance.Tracing, "Could not read job manager standard output", sr.Exception.Message);
return(Decision.Dontknow);
}
string firstline = sr.ReadLine();
if (sr.EndOfStream || firstline == null)
{
sr.Close();
return(Decision.No);
}
sr.Close();
if (firstline.Contains("Error parsing input XML file"))
{
this.Log(DiagnosisMessage.Importance.Final, "The job manager cannot parse the XML plan file.\n",
"This means probably that the version of LinqToDryad.dll that you are using does not match the XmlExecHost.exe file from your drop.");
return(Decision.Yes);
}
return(Decision.No);
}
/// <summary>
/// Detect whether vertex terminates with a stack overflow.
/// </summary>
/// <returns>True if this seems likely.</returns>
protected virtual Decision StackOverflow()
{
IClusterResidentObject stdout = this.Job.ClusterConfiguration.ProcessStdoutFile(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary);
if (stdout.Exception != null)
{
return(Decision.Dontknow);
}
ISharedStreamReader sr = stdout.GetStream();
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
if (line.Contains("StackOverflowException"))
{
this.Log(DiagnosisMessage.Importance.Final, "Error found in vertex stderr:", line);
sr.Close();
return(Decision.Yes);
}
}
sr.Close();
return(Decision.Dontknow);
}
/// <summary>
/// Load a specified file.
/// </summary>
/// <param name="file">File to load.</param>
public void LoadFile(IClusterResidentObject file)
{
string filename = file.Name;
bool text = true;
string basefilename = Path.GetFileName(filename);
if (basefilename != null && (basefilename.EndsWith(".log") && basefilename.StartsWith("cosmos")))
{
text = false;
}
long len = file.Size;
this.Initialize(text, basefilename);
//ISharedStreamReader sr = new FileSharedStreamReader(filename);
ISharedStreamReader sr = file.GetStream(false);
long lineno = 0;
long bytes = 0;
List <TextFileLine> toAddText = new List <TextFileLine>();
List <PositionedDryadLogEntry> toAddLog = new List <PositionedDryadLogEntry>();
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
bytes += line.Length;
if (this.shownText != null)
{
toAddText.Add(new TextFileLine(lineno, line));
}
else
{
PositionedDryadLogEntry cle = new PositionedDryadLogEntry(filename, lineno, line);
if (cle.Malformed)
{
Trace.TraceInformation("Malformed log entry: " + cle.OriginalLogLine);
}
else
{
toAddLog.Add(cle);
}
}
if (lineno++ % 100 == 0 && len > 0)
{
this.UpdateProgress((int)(bytes * 100 / len));
}
}
if (this.shownText != null)
{
this.shownText.SetItems(toAddText);
}
else
{
this.shownLogLines.SetItems(toAddLog);
}
this.Status("Loaded " + lineno + " lines.", StatusKind.OK);
this.UpdateProgress(100);
sr.Close();
this.filteredDataGridView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.DisplayedCellsExceptHeader);
}
/// <summary>
/// Scan the JM stdout looking for the specified vertex; display the lines in the file view.
/// Run in the background.
/// </summary>
/// <param name="vertex">Vertex to look for.</param>
/// <returns>true if the information was found.</returns>
/// <param name="logViewer">Viewer to use to display the logs.</param>
/// <param name="stdout">Job standard output stream.</param>
private static bool ScanJMStdout(ExecutedVertexInstance vertex, IClusterResidentObject stdout, LogViewer logViewer)
{
if (vertex == null || vertex.IsManager)
return false;
string vertexId = vertex.UniqueID;
string name = string.Format(@"\s{0}.{1}\s", vertex.Number, vertex.Version); // the dot could match a space too.
string regexstring = string.Format(@"vertex\s{0}\s(.*)\sv.{1}\s|", vertex.Number, vertex.Version);
if (vertexId != "")
regexstring += vertexId + "|";
regexstring += name + "|" + vertex.UniqueID;
Regex regex = new Regex(regexstring, RegexOptions.Compiled);
Trace.TraceInformation(regex.ToString());
long length = stdout.Size;
logViewer.Status("Looking for " + vertex.Name, StatusKind.LongOp);
if (length == 0)
{
logViewer.Status("JM stdout is empty.", StatusKind.Error);
logViewer.Done();
return false;
}
ISharedStreamReader sr = stdout.GetStream();
if (sr.Exception != null)
{
logViewer.Status("Error opening JM stdout: " + sr.Exception.Message, StatusKind.Error);
logViewer.Done();
return false;
}
try
{
long read = 0;
long lines = 0;
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
read += line.Length;
if (regex.IsMatch(line))
logViewer.AddLine(stdout.ToString(), lines, line);
lines++;
if (lines % 100 == 0 && length > 0)
{
if (logViewer.Cancelled)
break;
logViewer.UpdateProgress(Math.Min((int)(read * 100 / length), 100)); // the length can grow while the file is being read
}
}
sr.Close();
}
finally
{
logViewer.Done();
}
return true;
}
/// <summary>
/// Get the contents of a specified cluster resident object.
/// </summary>
/// <param name="path">Cluster object whose contents is read.</param>
/// <param name="pattern">Pattern to filter contents, for folders.</param>
/// <returns>The file contents.</returns>
/// <param name="manager">Communication manager.</param>
private static FileContents GetContents(CommManager manager, IClusterResidentObject path, string pattern)
{
if (path == null)
{
return new FileContents("Null path");
}
StringBuilder output = new StringBuilder();
Dictionary<string, IClusterResidentObject> linkCache = new Dictionary<string, IClusterResidentObject>();
linkCache.Add(path.ToString(), path);
string error = (path.RepresentsAFolder ? "Folder " : "") + path;
if (path.Exception != null)
{
error += " [Error accessing: " + path.Exception.Message + "]";
return new FileContents(error);
}
if (path.RepresentsAFolder)
{
IEnumerable<IClusterResidentObject> dirs = path.GetFilesAndFolders(pattern);
int displayed = 0;
foreach (IClusterResidentObject d in dirs)
{
manager.Token.ThrowIfCancellationRequested();
if (d.Exception != null)
{
error += " [Error " + d.Exception.Message + "]";
return new FileContents(error);
}
if (d.RepresentsAFolder)
{
string dirdisplay = string.Format("{0:u} {1,16} file://{2}", d.CreationTime, "d", d.Name);
output.AppendLine(dirdisplay);
}
else
{
string filedisplay = string.Format("{0:u} {1,16:N0} file://{2}", d.CreationTime, d.Size, d.Name);
output.AppendLine(filedisplay);
}
linkCache.Add("file://" + d.Name, d);
displayed++;
}
if (displayed == 0)
error += "[empty]";
return new FileContents(output.ToString(), error, linkCache);
}
else
{
manager.Status("Extracting contents of " + path, StatusKind.LongOp);
ISharedStreamReader sr = path.GetStream();
if (sr.Exception != null)
{
error += " [Error " + sr.Exception.Message + "]";
return new FileContents(error);
}
else
{
if (path.Size == 0)
error += "[empty]";
var contents = sr.ReadToEnd(manager.Token);
return new FileContents(contents, error, linkCache);
}
}
}
/// <summary>
/// Parse a log file of the job manager and extract useful information.
/// </summary>
/// <param name="logfile">Log file to parse.</param>
/// <param name="statusReporter">Delegate used to parse errors.</param>
/// <returns>True if parsing succeeds.</returns>
internal bool ParseJMLogFile(IClusterResidentObject logfile, StatusReporter statusReporter)
{
bool success = true;
ISharedStreamReader sr = logfile.GetStream();
if (sr.Exception != null)
{
statusReporter("Exception while opening file " + logfile + ":" + sr.Exception.Message, StatusKind.Error);
return false;
}
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
if (!line.Contains("DryadProfiler")) continue;
DryadLogEntry le = new DryadLogEntry(line);
if (le.Subsystem != "DryadProfiler") continue;
if (!le.Message.EndsWith("channel status")) continue;
Dictionary<string, string> kvp = Utilities.ParseCommaSeparatedKeyValuePair(le.ExtraInfo);
string verver = kvp["Vertex"];
string[] numbers = verver.Split('.');
int vertex = int.Parse(numbers[0]);
int version = int.Parse(numbers[1]);
ExecutedVertexInstance vi = this.jobVertices.FindVertex(vertex, version);
if (vi == null)
{
// We have overshot the information about the vertices parsed from stdout; stop parsing here
success = false;
break;
}
if (le.Message == "Input channel status")
{
// Vertex=69.0, Name=Merge__446[3], MachPod=sherwood-005:pod1, TotalRead=1470802, TotalReadFromMach=1470802, TotalReadCrossMach=1470802, TotalReadCrossPod=0
long info = long.Parse(kvp["TotalRead"]);
vi.DataRead = info;
}
else if (le.Message == "Output channel status")
{
// Vertex=69.0, Name=Merge__446[3], MachPod=sherwood-005:pod1, TotalWrite=1213418
long info = long.Parse(kvp["TotalWrite"]);
vi.DataWritten = info;
}
}
sr.Close();
return success;
}
/// <summary>
/// Parse the stdout.txt file from the job manager.
/// </summary>
/// <param name="file">File to parse.</param>
/// <param name="manager">Communication manager.</param>
/// <returns>True if the parsing succeeds.</returns>
private bool ParseStdout(IClusterResidentObject file, CommManager manager)
{
int currentLine = 0;
if (this.stdoutLinesParsed == 0)
// don't lose it if we are only parsing the tail.
this.lastTimestampSeen = this.Summary.Date; // start from the job submission timestamp
// we are reusing the stream
this.stdoutLinesParsed = 0;
try
{
long filesize = file.Size;
long readbytes = 0;
string message = "Scanning JM stdout " + file;
if (filesize >= 0)
message += string.Format("({0:N0} bytes)", filesize);
manager.Status(message, StatusKind.LongOp);
if (this.cachedStdoutReader == null)
this.cachedStdoutReader = file.GetStream();
if (this.cachedStdoutReader.Exception != null)
{
manager.Status("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error);
return false;
}
while (!this.cachedStdoutReader.EndOfStream)
{
string line = this.cachedStdoutReader.ReadLine();
readbytes += line.Length;
if (currentLine >= this.stdoutLinesParsed)
{
while (true)
{
manager.Token.ThrowIfCancellationRequested();
int startLine = currentLine;
bool completeLine = true;
try
{
completeLine = this.ParseStdoutLineNew(line);
}
catch (Exception ex)
{
manager.Status(string.Format("Line {0}: Exception {1}", currentLine, ex.Message), StatusKind.Error);
Console.WriteLine("Line {0}: Exception {1}", currentLine, ex);
}
if (!completeLine)
{
if (this.cachedStdoutReader.EndOfStream)
{
throw new Exception("File ended while scanning for closing quote started on line " + startLine);
}
string extraline = this.cachedStdoutReader.ReadLine();
line += "\n" + extraline;
currentLine++;
}
else break;
}
}
currentLine++;
if (currentLine % 100 == 0 && filesize > 0)
{
manager.Progress(Math.Min(100, (int)(100 * readbytes / filesize)));
}
}
this.stdoutLinesParsed = currentLine;
if (this.ManagerVertex != null)
{
if (this.ManagerVertex.End == DateTime.MinValue)
// approximation
this.ManagerVertex.End = this.lastTimestampSeen;
// we are done with this stream
if (this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Failed ||
this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Successful)
{
this.cachedStdoutReader.Close();
this.cachedStdoutReader = null; // will force reopening if refreshed
}
}
return true;
}
catch (Exception e)
{
manager.Status("Exception while reading stdout " + e.Message, StatusKind.Error);
Trace.TraceInformation(e.ToString());
return false;
}
}