/// <summary>
/// Try to diagnose whether there's a CLR mismatch error.
/// </summary>
/// <returns>True if this is the problem.</returns>
protected bool CLRStartupProblems()
{
IClusterResidentObject stdout = this.Job.ClusterConfiguration.ProcessStdoutFile(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary);
if (stdout.Exception != null)
{
return(false);
}
ISharedStreamReader sr = stdout.GetStream();
// only look for the error in the first 10 lines
for (int i = 0; i < 10; i++)
{
if (sr.EndOfStream)
{
sr.Close();
return(false);
}
string line = sr.ReadLine();
if (line.Contains("Error code 2148734720 (0x80131700)"))
{
this.Log(DiagnosisMessage.Importance.Final, "Error found in vertex stdout:", line);
sr.Close();
return(true);
}
}
sr.Close();
return(false);
}
/// <summary>
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
/// </summary>
/// <returns>A stream that can be used to access the object contents.</returns>
public override ISharedStreamReader GetStream()
{
ISharedStreamReader baseStream = base.GetStream();
if (baseStream != null)
{
// file is cached
Trace.TraceInformation("Reading from local cache {0}", baseStream);
return(baseStream);
}
Stream stream;
if (this.IsDfsStream)
{
var dfsFileStream = this.client.GetDfsFileStream(this.path);
stream = dfsFileStream.Stream;
}
else
{
stream = new AzureLogReaderStream(
this.client.AccountName,
this.client.AccountKey,
this.client.ContainerName,
this.path);
}
long size = this.Size;
int bufferSize = 1024 * 1024;
if (size >= 0)
{
bufferSize = (int)(size / 10);
if (bufferSize < 1024 * 1024)
{
bufferSize = 1024 * 1024;
}
if (bufferSize > 20 * 1024 * 1024)
{
bufferSize = 20 * 1024 * 1024;
}
}
StreamReader reader = new StreamReader(stream, System.Text.Encoding.UTF8, false, bufferSize);
if (this.ShouldCacheLocally && this.LocalCachePath != null)
{
// cache it
if (this.RepresentsAFolder)
{
throw new ClusterException("Cannot cache folders");
}
StreamWriter writer = this.CreateTempStream();
return(new SharedStreamReader(reader, writer, this.OnClose));
}
else
{
// dont cache it
return(new SharedStreamReader(reader));
}
}
/// <summary>
/// The stack trace of the vertex at the time of the crash.
/// </summary>
/// <returns>The stack trace or an empty collection.</returns>
public virtual IEnumerable <string> StackTrace()
{
IClusterResidentObject logdir = this.Job.ClusterConfiguration.ProcessWorkDirectory(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary);
IClusterResidentObject stackTrace = logdir.GetFile(this.stackTraceFile);
ISharedStreamReader sr = stackTrace.GetStream();
if (sr.Exception == null)
{
while (!sr.EndOfStream)
{
yield return(sr.ReadLine());
}
}
else
{
yield break;
}
}
/// <summary>
/// Discover whether the failure is caused by the inability to parse the XML plan.
/// </summary>
/// <returns>The decision.</returns>
public Decision XmlPlanParseError()
{
if (this.jobManager == null)
{
this.Log(DiagnosisMessage.Importance.Tracing, "Could not find job manager vertex information", "");
return(Decision.Dontknow);
}
IClusterResidentObject jmstdout = this.jobManager.StdoutFile;
if (jmstdout.Exception != null)
{
this.Log(DiagnosisMessage.Importance.Tracing, "Could not find job manager standard output", "");
return(Decision.Dontknow);
}
ISharedStreamReader sr = jmstdout.GetStream();
if (sr.Exception != null)
{
this.Log(DiagnosisMessage.Importance.Tracing, "Could not read job manager standard output", sr.Exception.Message);
return(Decision.Dontknow);
}
string firstline = sr.ReadLine();
if (sr.EndOfStream || firstline == null)
{
sr.Close();
return(Decision.No);
}
sr.Close();
if (firstline.Contains("Error parsing input XML file"))
{
this.Log(DiagnosisMessage.Importance.Final, "The job manager cannot parse the XML plan file.\n",
"This means probably that the version of LinqToDryad.dll that you are using does not match the XmlExecHost.exe file from your drop.");
return(Decision.Yes);
}
return(Decision.No);
}
/// <summary>
/// The stream with the file contents.
/// </summary>
/// <returns>A stream reder.</returns>
public override ISharedStreamReader GetStream()
{
try
{
if (!this.RepresentsAFolder)
{
//this.LocalCachePath = this.CachePath(this.Pathname);
ISharedStreamReader baseStream = base.GetStream();
if (baseStream != null)
{
// file is cached
Trace.TraceInformation("Reading from local cache {0}", baseStream);
return(baseStream);
}
}
if (this.LocalCachePath != null && this.ShouldCacheLocally)
{
// cache it
if (this.RepresentsAFolder)
{
throw new ClusterException("Cannot cache folders");
}
StreamWriter writer = this.CreateTempStream();
return(new FileSharedStreamReader(this.Pathname.ToString(), writer, this.OnClose));
}
else
{
// dont cache it
return(new FileSharedStreamReader(this.Pathname.ToString()));
}
}
catch (Exception ex)
{
return(new FileSharedStreamReader(ex));
}
}
/// <summary>
/// Detect whether vertex terminates with a stack overflow.
/// </summary>
/// <returns>True if this seems likely.</returns>
protected virtual Decision StackOverflow()
{
IClusterResidentObject stdout = this.Job.ClusterConfiguration.ProcessStdoutFile(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary);
if (stdout.Exception != null)
{
return(Decision.Dontknow);
}
ISharedStreamReader sr = stdout.GetStream();
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
if (line.Contains("StackOverflowException"))
{
this.Log(DiagnosisMessage.Importance.Final, "Error found in vertex stderr:", line);
sr.Close();
return(Decision.Yes);
}
}
sr.Close();
return(Decision.Dontknow);
}
/// <summary>
/// Load a specified file.
/// </summary>
/// <param name="file">File to load.</param>
public void LoadFile(IClusterResidentObject file)
{
string filename = file.Name;
bool text = true;
string basefilename = Path.GetFileName(filename);
if (basefilename != null && (basefilename.EndsWith(".log") && basefilename.StartsWith("cosmos")))
{
text = false;
}
long len = file.Size;
this.Initialize(text, basefilename);
//ISharedStreamReader sr = new FileSharedStreamReader(filename);
ISharedStreamReader sr = file.GetStream(false);
long lineno = 0;
long bytes = 0;
List <TextFileLine> toAddText = new List <TextFileLine>();
List <PositionedDryadLogEntry> toAddLog = new List <PositionedDryadLogEntry>();
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
bytes += line.Length;
if (this.shownText != null)
{
toAddText.Add(new TextFileLine(lineno, line));
}
else
{
PositionedDryadLogEntry cle = new PositionedDryadLogEntry(filename, lineno, line);
if (cle.Malformed)
{
Trace.TraceInformation("Malformed log entry: " + cle.OriginalLogLine);
}
else
{
toAddLog.Add(cle);
}
}
if (lineno++ % 100 == 0 && len > 0)
{
this.UpdateProgress((int)(bytes * 100 / len));
}
}
if (this.shownText != null)
{
this.shownText.SetItems(toAddText);
}
else
{
this.shownLogLines.SetItems(toAddLog);
}
this.Status("Loaded " + lineno + " lines.", StatusKind.OK);
this.UpdateProgress(100);
sr.Close();
this.filteredDataGridView.DataGridView.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.DisplayedCellsExceptHeader);
}
/// <summary>
/// Parse a part of the 'originalInfo.txt' file to discover a set of channel endpoints.
/// </summary>
/// <param name="sr">Stream reader which contains the channel information.</param>
/// <returns>The list of channels, or null on failure.</returns>
/// <param name="uriprefix">If the channel is an output, prefix the path with this; this is null for inputs.</param>
/// <param name="skip">If true, do not return anything (still useful to advance the stream reader).</param>
/// <param name="fast">If true the channel sizes are not discovered; this is much faster, since no remote machines are queried for files.</param>
/// <param name="manager">Communication manager.</param>
private Dictionary<int, ChannelEndpointDescription> DiscoverOriginalInfoChannels(ISharedStreamReader sr, string uriprefix, bool skip, bool fast, CommManager manager)
{
bool isInput = uriprefix == null;
string countline = sr.ReadLine();
if (countline == null)
return null;
int channelCount;
int spaceIndex = countline.IndexOf(' ');
if (spaceIndex > 0)
countline = countline.Substring(0, spaceIndex);
bool success = int.TryParse(countline, out channelCount);
if (!success)
return null;
var channels = new Dictionary<int, ChannelEndpointDescription>(channelCount);
for (int i = 0; i < channelCount; i++)
{
string channel = sr.ReadLine();
if (channel == null)
{
manager.Progress(100);
return null;
}
if (!skip)
{
ChannelEndpointDescription desc = new ChannelEndpointDescription(isInput, i, channel, uriprefix, fast, manager.Status);
channels.Add(i, desc);
manager.Progress(i * 100 / channelCount);
}
}
manager.Progress(100);
if (skip)
return null;
return channels;
}
/// <summary>
/// Create a ScopeJobStaticPlan.
/// </summary>
/// <param name="config">Cluster configuration.</param>
/// <param name="planFile">Stream containing the file.</param>
/// <param name="vertexDef">File containing the vertex definition (ScopeVertexDef.xml).</param>
// ReSharper disable once UnusedParameter.Local
public ScopeJobStaticPlan(ClusterConfiguration config, ISharedStreamReader planFile, ISharedStreamReader vertexDef)
: base(planFile)
{
this.vertexDef = vertexDef;
}
/// <summary>
/// Create a DryadLinqJobStaticPlan.
/// </summary>
/// <param name="config">Cluster configuration.</param>
/// <param name="planFile">Stream containing the file.</param>
// ReSharper disable once UnusedParameter.Local
public DryadLinqJobStaticPlan(ClusterConfiguration config, ISharedStreamReader planFile)
: base(planFile)
{
}
/// <summary>
/// Create a dryadlinq job plan starting from an xml plan file.
/// </summary>
/// <param name="plan">Stream containing the plan.</param>
protected DryadJobStaticPlan(ISharedStreamReader plan)
{
if (plan.Exception != null)
// don't do this
throw plan.Exception;
this.planStream = plan;
this.stages = new Dictionary<int, Stage>();
this.connections = new List<Connection>();
this.perNodeConnectionInfo = new Dictionary<int, ConnectionInformation>();
this.fictitiousStages = 0;
}
/// <summary>
/// Parse the stdout.txt file from the job manager.
/// </summary>
/// <param name="file">File to parse.</param>
/// <param name="manager">Communication manager.</param>
/// <returns>True if the parsing succeeds.</returns>
private bool ParseStdout(IClusterResidentObject file, CommManager manager)
{
int currentLine = 0;
if (this.stdoutLinesParsed == 0)
// don't lose it if we are only parsing the tail.
this.lastTimestampSeen = this.Summary.Date; // start from the job submission timestamp
// we are reusing the stream
this.stdoutLinesParsed = 0;
try
{
long filesize = file.Size;
long readbytes = 0;
string message = "Scanning JM stdout " + file;
if (filesize >= 0)
message += string.Format("({0:N0} bytes)", filesize);
manager.Status(message, StatusKind.LongOp);
if (this.cachedStdoutReader == null)
this.cachedStdoutReader = file.GetStream();
if (this.cachedStdoutReader.Exception != null)
{
manager.Status("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error);
return false;
}
while (!this.cachedStdoutReader.EndOfStream)
{
string line = this.cachedStdoutReader.ReadLine();
readbytes += line.Length;
if (currentLine >= this.stdoutLinesParsed)
{
while (true)
{
manager.Token.ThrowIfCancellationRequested();
int startLine = currentLine;
bool completeLine = true;
try
{
completeLine = this.ParseStdoutLineNew(line);
}
catch (Exception ex)
{
manager.Status(string.Format("Line {0}: Exception {1}", currentLine, ex.Message), StatusKind.Error);
Console.WriteLine("Line {0}: Exception {1}", currentLine, ex);
}
if (!completeLine)
{
if (this.cachedStdoutReader.EndOfStream)
{
throw new Exception("File ended while scanning for closing quote started on line " + startLine);
}
string extraline = this.cachedStdoutReader.ReadLine();
line += "\n" + extraline;
currentLine++;
}
else break;
}
}
currentLine++;
if (currentLine % 100 == 0 && filesize > 0)
{
manager.Progress(Math.Min(100, (int)(100 * readbytes / filesize)));
}
}
this.stdoutLinesParsed = currentLine;
if (this.ManagerVertex != null)
{
if (this.ManagerVertex.End == DateTime.MinValue)
// approximation
this.ManagerVertex.End = this.lastTimestampSeen;
// we are done with this stream
if (this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Failed ||
this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Successful)
{
this.cachedStdoutReader.Close();
this.cachedStdoutReader = null; // will force reopening if refreshed
}
}
return true;
}
catch (Exception e)
{
manager.Status("Exception while reading stdout " + e.Message, StatusKind.Error);
Trace.TraceInformation(e.ToString());
return false;
}
}
