/// <summary> /// Try to diagnose whether there's a CLR mismatch error. /// </summary> /// <returns>True if this is the problem.</returns> protected bool CLRStartupProblems() { IClusterResidentObject stdout = this.Job.ClusterConfiguration.ProcessStdoutFile(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary); if (stdout.Exception != null) { return(false); } ISharedStreamReader sr = stdout.GetStream(); // only look for the error in the first 10 lines for (int i = 0; i < 10; i++) { if (sr.EndOfStream) { sr.Close(); return(false); } string line = sr.ReadLine(); if (line.Contains("Error code 2148734720 (0x80131700)")) { this.Log(DiagnosisMessage.Importance.Final, "Error found in vertex stdout:", line); sr.Close(); return(true); } } sr.Close(); return(false); }
/// <summary> /// The stack trace of the vertex at the time of the crash. /// </summary> /// <returns>The stack trace or an empty collection.</returns> public virtual IEnumerable <string> StackTrace() { IClusterResidentObject logdir = this.Job.ClusterConfiguration.ProcessWorkDirectory(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary); IClusterResidentObject stackTrace = logdir.GetFile(this.stackTraceFile); ISharedStreamReader sr = stackTrace.GetStream(); if (sr.Exception == null) { while (!sr.EndOfStream) { yield return(sr.ReadLine()); } } else { yield break; } }
/// <summary> /// Discover whether the failure is caused by the inability to parse the XML plan. /// </summary> /// <returns>The decision.</returns> public Decision XmlPlanParseError() { if (this.jobManager == null) { this.Log(DiagnosisMessage.Importance.Tracing, "Could not find job manager vertex information", ""); return(Decision.Dontknow); } IClusterResidentObject jmstdout = this.jobManager.StdoutFile; if (jmstdout.Exception != null) { this.Log(DiagnosisMessage.Importance.Tracing, "Could not find job manager standard output", ""); return(Decision.Dontknow); } ISharedStreamReader sr = jmstdout.GetStream(); if (sr.Exception != null) { this.Log(DiagnosisMessage.Importance.Tracing, "Could not read job manager standard output", sr.Exception.Message); return(Decision.Dontknow); } string firstline = sr.ReadLine(); if (sr.EndOfStream || firstline == null) { sr.Close(); return(Decision.No); } sr.Close(); if (firstline.Contains("Error parsing input XML file")) { this.Log(DiagnosisMessage.Importance.Final, "The job manager cannot parse the XML plan file.\n", "This means probably that the version of LinqToDryad.dll that you are using does not match the XmlExecHost.exe file from your drop."); return(Decision.Yes); } return(Decision.No); }
/// <summary> /// Detect whether vertex terminates with a stack overflow. /// </summary> /// <returns>True if this seems likely.</returns> protected virtual Decision StackOverflow() { IClusterResidentObject stdout = this.Job.ClusterConfiguration.ProcessStdoutFile(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary); if (stdout.Exception != null) { return(Decision.Dontknow); } ISharedStreamReader sr = stdout.GetStream(); while (!sr.EndOfStream) { string line = sr.ReadLine(); if (line.Contains("StackOverflowException")) { this.Log(DiagnosisMessage.Importance.Final, "Error found in vertex stderr:", line); sr.Close(); return(Decision.Yes); } } sr.Close(); return(Decision.Dontknow); }
/// <summary> /// Load a specified file. /// </summary> /// <param name="file">File to load.</param> public void LoadFile(IClusterResidentObject file) { string filename = file.Name; bool text = true; string basefilename = Path.GetFileName(filename); if (basefilename != null && (basefilename.EndsWith(".log") && basefilename.StartsWith("cosmos"))) { text = false; } long len = file.Size; this.Initialize(text, basefilename); //ISharedStreamReader sr = new FileSharedStreamReader(filename); ISharedStreamReader sr = file.GetStream(false); long lineno = 0; long bytes = 0; List <TextFileLine> toAddText = new List <TextFileLine>(); List <PositionedDryadLogEntry> toAddLog = new List <PositionedDryadLogEntry>(); while (!sr.EndOfStream) { string line = sr.ReadLine(); bytes += line.Length; if (this.shownText != null) { toAddText.Add(new TextFileLine(lineno, line)); } else { PositionedDryadLogEntry cle = new PositionedDryadLogEntry(filename, lineno, line); if (cle.Malformed) { Trace.TraceInformation("Malformed log entry: " + cle.OriginalLogLine); } else { toAddLog.Add(cle); } } if (lineno++ % 100 == 0 && len > 0) { this.UpdateProgress((int)(bytes * 100 / len)); } } if (this.shownText != null) { this.shownText.SetItems(toAddText); } else { this.shownLogLines.SetItems(toAddLog); } this.Status("Loaded " + lineno + " lines.", StatusKind.OK); this.UpdateProgress(100); sr.Close(); this.filteredDataGridView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.DisplayedCellsExceptHeader); }
/// <summary> /// Scan the JM stdout looking for the specified vertex; display the lines in the file view. /// Run in the background. /// </summary> /// <param name="vertex">Vertex to look for.</param> /// <returns>true if the information was found.</returns> /// <param name="logViewer">Viewer to use to display the logs.</param> /// <param name="stdout">Job standard output stream.</param> private static bool ScanJMStdout(ExecutedVertexInstance vertex, IClusterResidentObject stdout, LogViewer logViewer) { if (vertex == null || vertex.IsManager) return false; string vertexId = vertex.UniqueID; string name = string.Format(@"\s{0}.{1}\s", vertex.Number, vertex.Version); // the dot could match a space too. string regexstring = string.Format(@"vertex\s{0}\s(.*)\sv.{1}\s|", vertex.Number, vertex.Version); if (vertexId != "") regexstring += vertexId + "|"; regexstring += name + "|" + vertex.UniqueID; Regex regex = new Regex(regexstring, RegexOptions.Compiled); Trace.TraceInformation(regex.ToString()); long length = stdout.Size; logViewer.Status("Looking for " + vertex.Name, StatusKind.LongOp); if (length == 0) { logViewer.Status("JM stdout is empty.", StatusKind.Error); logViewer.Done(); return false; } ISharedStreamReader sr = stdout.GetStream(); if (sr.Exception != null) { logViewer.Status("Error opening JM stdout: " + sr.Exception.Message, StatusKind.Error); logViewer.Done(); return false; } try { long read = 0; long lines = 0; while (!sr.EndOfStream) { string line = sr.ReadLine(); read += line.Length; if (regex.IsMatch(line)) logViewer.AddLine(stdout.ToString(), lines, line); lines++; if (lines % 100 == 0 && length > 0) { if (logViewer.Cancelled) break; logViewer.UpdateProgress(Math.Min((int)(read * 100 / length), 100)); // the length can grow while the file is being read } } sr.Close(); } finally { logViewer.Done(); } return true; }
/// <summary> /// Get the contents of a specified cluster resident object. /// </summary> /// <param name="path">Cluster object whose contents is read.</param> /// <param name="pattern">Pattern to filter contents, for folders.</param> /// <returns>The file contents.</returns> /// <param name="manager">Communication manager.</param> private static FileContents GetContents(CommManager manager, IClusterResidentObject path, string pattern) { if (path == null) { return new FileContents("Null path"); } StringBuilder output = new StringBuilder(); Dictionary<string, IClusterResidentObject> linkCache = new Dictionary<string, IClusterResidentObject>(); linkCache.Add(path.ToString(), path); string error = (path.RepresentsAFolder ? "Folder " : "") + path; if (path.Exception != null) { error += " [Error accessing: " + path.Exception.Message + "]"; return new FileContents(error); } if (path.RepresentsAFolder) { IEnumerable<IClusterResidentObject> dirs = path.GetFilesAndFolders(pattern); int displayed = 0; foreach (IClusterResidentObject d in dirs) { manager.Token.ThrowIfCancellationRequested(); if (d.Exception != null) { error += " [Error " + d.Exception.Message + "]"; return new FileContents(error); } if (d.RepresentsAFolder) { string dirdisplay = string.Format("{0:u} {1,16} file://{2}", d.CreationTime, "d", d.Name); output.AppendLine(dirdisplay); } else { string filedisplay = string.Format("{0:u} {1,16:N0} file://{2}", d.CreationTime, d.Size, d.Name); output.AppendLine(filedisplay); } linkCache.Add("file://" + d.Name, d); displayed++; } if (displayed == 0) error += "[empty]"; return new FileContents(output.ToString(), error, linkCache); } else { manager.Status("Extracting contents of " + path, StatusKind.LongOp); ISharedStreamReader sr = path.GetStream(); if (sr.Exception != null) { error += " [Error " + sr.Exception.Message + "]"; return new FileContents(error); } else { if (path.Size == 0) error += "[empty]"; var contents = sr.ReadToEnd(manager.Token); return new FileContents(contents, error, linkCache); } } }
/// <summary> /// Parse a log file of the job manager and extract useful information. /// </summary> /// <param name="logfile">Log file to parse.</param> /// <param name="statusReporter">Delegate used to parse errors.</param> /// <returns>True if parsing succeeds.</returns> internal bool ParseJMLogFile(IClusterResidentObject logfile, StatusReporter statusReporter) { bool success = true; ISharedStreamReader sr = logfile.GetStream(); if (sr.Exception != null) { statusReporter("Exception while opening file " + logfile + ":" + sr.Exception.Message, StatusKind.Error); return false; } while (!sr.EndOfStream) { string line = sr.ReadLine(); if (!line.Contains("DryadProfiler")) continue; DryadLogEntry le = new DryadLogEntry(line); if (le.Subsystem != "DryadProfiler") continue; if (!le.Message.EndsWith("channel status")) continue; Dictionary<string, string> kvp = Utilities.ParseCommaSeparatedKeyValuePair(le.ExtraInfo); string verver = kvp["Vertex"]; string[] numbers = verver.Split('.'); int vertex = int.Parse(numbers[0]); int version = int.Parse(numbers[1]); ExecutedVertexInstance vi = this.jobVertices.FindVertex(vertex, version); if (vi == null) { // We have overshot the information about the vertices parsed from stdout; stop parsing here success = false; break; } if (le.Message == "Input channel status") { // Vertex=69.0, Name=Merge__446[3], MachPod=sherwood-005:pod1, TotalRead=1470802, TotalReadFromMach=1470802, TotalReadCrossMach=1470802, TotalReadCrossPod=0 long info = long.Parse(kvp["TotalRead"]); vi.DataRead = info; } else if (le.Message == "Output channel status") { // Vertex=69.0, Name=Merge__446[3], MachPod=sherwood-005:pod1, TotalWrite=1213418 long info = long.Parse(kvp["TotalWrite"]); vi.DataWritten = info; } } sr.Close(); return success; }
/// <summary> /// Parse the stdout.txt file from the job manager. /// </summary> /// <param name="file">File to parse.</param> /// <param name="manager">Communication manager.</param> /// <returns>True if the parsing succeeds.</returns> private bool ParseStdout(IClusterResidentObject file, CommManager manager) { int currentLine = 0; if (this.stdoutLinesParsed == 0) // don't lose it if we are only parsing the tail. this.lastTimestampSeen = this.Summary.Date; // start from the job submission timestamp // we are reusing the stream this.stdoutLinesParsed = 0; try { long filesize = file.Size; long readbytes = 0; string message = "Scanning JM stdout " + file; if (filesize >= 0) message += string.Format("({0:N0} bytes)", filesize); manager.Status(message, StatusKind.LongOp); if (this.cachedStdoutReader == null) this.cachedStdoutReader = file.GetStream(); if (this.cachedStdoutReader.Exception != null) { manager.Status("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error); return false; } while (!this.cachedStdoutReader.EndOfStream) { string line = this.cachedStdoutReader.ReadLine(); readbytes += line.Length; if (currentLine >= this.stdoutLinesParsed) { while (true) { manager.Token.ThrowIfCancellationRequested(); int startLine = currentLine; bool completeLine = true; try { completeLine = this.ParseStdoutLineNew(line); } catch (Exception ex) { manager.Status(string.Format("Line {0}: Exception {1}", currentLine, ex.Message), StatusKind.Error); Console.WriteLine("Line {0}: Exception {1}", currentLine, ex); } if (!completeLine) { if (this.cachedStdoutReader.EndOfStream) { throw new Exception("File ended while scanning for closing quote started on line " + startLine); } string extraline = this.cachedStdoutReader.ReadLine(); line += "\n" + extraline; currentLine++; } else break; } } currentLine++; if (currentLine % 100 == 0 && filesize > 0) { manager.Progress(Math.Min(100, (int)(100 * readbytes / filesize))); } } this.stdoutLinesParsed = currentLine; if (this.ManagerVertex != null) { if (this.ManagerVertex.End == DateTime.MinValue) // approximation this.ManagerVertex.End = this.lastTimestampSeen; // we are done with this stream if (this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Failed || this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Successful) { this.cachedStdoutReader.Close(); this.cachedStdoutReader = null; // will force reopening if refreshed } } return true; } catch (Exception e) { manager.Status("Exception while reading stdout " + e.Message, StatusKind.Error); Trace.TraceInformation(e.ToString()); return false; } }