/// <summary>
/// Discover the vertex channels in a Scope-generated vcmdStart*xml file.
/// </summary>
/// <returns>True if the discovery was successful.</returns>
/// <param name="inputs">If true discover the inputs.</param>
/// <param name="outputs">If true discover the outputs.</param>
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
/// <param name="manager">Communication manager.</param>
// ReSharper disable UnusedParameter.Global
public bool DiscoverScopeChannels(bool inputs, bool outputs, bool fast, CommManager manager)
// ReSharper restore UnusedParameter.Global
{
// find the xml file
var files = this.WorkDirectory.GetFilesAndFolders("vcmdStart*.xml").ToList();
if (files.Count != 1)
{
manager.Status("Cannot locate vcmdStart*.xml file", StatusKind.Error);
return false;
}
ISharedStreamReader sr = files.First().GetStream();
if (sr.Exception != null)
{
manager.Status("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error);
return false;
}
// ReSharper disable PossibleNullReferenceException
XDocument plan = XDocument.Parse(sr.ReadToEnd(manager.Token));
if (inputs && this.InputChannels == null)
{
var channels = new Dictionary<int, ChannelEndpointDescription>();
IEnumerable<XElement> inputsData = plan.Root.Element("inputs").Elements();
int chno = 0;
foreach (var e in inputsData)
{
string chpath = e.Attribute("path").Value;
long size = long.Parse(e.Attribute("length").Value);
ChannelEndpointDescription desc = new ChannelEndpointDescription(true, chno, chpath, size);
channels.Add(chno, desc);
chno++;
}
this.InputChannels = channels;
}
if (outputs && this.OutputChannels == null)
{
var channels = new Dictionary<int, ChannelEndpointDescription>();
IEnumerable<XElement> inputsData = plan.Root.Element("outputs").Elements();
int chno = 0;
foreach (var e in inputsData)
{
string chpath = e.Attribute("path").Value;
ChannelEndpointDescription desc = new ChannelEndpointDescription(true, chno, chpath, -1);
channels.Add(chno, desc);
chno ++;
}
this.OutputChannels = channels;
}
// ReSharper restore PossibleNullReferenceException
sr.Close();
return true;
}
/// <summary>
/// New JM stdout parsing code, for YARN-based DryadLINQ.
/// </summary>
/// <param name="line">Line to parse.</param>
/// <returns>False if the line terminated in a quoted string and has to be combined with the next line.</returns>
private bool ParseStdoutLineNew(string line)
{
if (string.IsNullOrWhiteSpace(line)) return true;
Dictionary<string, string> kvp = Utilities.ParseCSVKVP(line);
if (kvp == null) return false;
var strTs = kvp["logtimelocal"];
int cutOff = strTs.IndexOf("UTC");
if (cutOff >= 0)
{
strTs = strTs.Substring(0, cutOff);
}
DateTime timeStamp = DateTime.Parse(strTs, CultureInfo.InvariantCulture);
timeStamp = timeStamp.ToLocalTime();
this.lastTimestampSeen = timeStamp;
if (kvp.ContainsKey("job"))
{
string operation = kvp["job"];
switch (operation)
{
case "start":
this.ManagerVertex.SetStartInformation(this, this.Summary.Machine, timeStamp, this.Summary.ManagerProcessGuid, "");
this.ManagerVertex.StartCommandTime = this.ManagerVertex.CreationTime = this.ManagerVertex.VertexScheduleTime = timeStamp;
break;
case "stop":
this.ManagerVertex.End = timeStamp;
string exitcode;
if (kvp.TryGetValue("exitcode", out exitcode))
{
this.ErrorCode = exitcode;
int numCode = Convert.ToInt32(exitcode, 16);
if (numCode == 0)
{
this.ManagerVertex.SetState(ExecutedVertexInstance.VertexState.Successful);
}
else
{
this.ManagerVertex.SetState(ExecutedVertexInstance.VertexState.Failed);
}
}
string errorstring;
if (kvp.TryGetValue("errorstring", out errorstring))
{
this.ManagerVertex.AddErrorString(errorstring);
this.AbortingMsg = errorstring;
}
break;
}
}
else if (kvp.ContainsKey("vertex"))
{
string vertex = kvp["vertex"];
int number;
int version;
int dot = vertex.IndexOf('.');
if (dot < 0)
{
number = int.Parse(vertex);
version = int.Parse(kvp["version"]);
}
else
{
number = int.Parse(vertex.Substring(0, dot));
version = int.Parse(vertex.Substring(dot + 1));
}
if (kvp.ContainsKey("transition"))
{
string transition = kvp["transition"];
switch (transition)
{
case "created":
{
string name = kvp["name"];
ExecutedVertexInstance vi = new ExecutedVertexInstance(this, number, version, name, "", timeStamp);
this.jobVertices.Add(vi);
}
break;
case "starting":
{
// not doing anything
break;
}
case "running":
{
string process;
kvp.TryGetValue("id", out process);
if (process == null)
kvp.TryGetValue("process", out process);
string machine = kvp["computer"];
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
this.jobVertices.Remap(vi, process);
string pid = this.ClusterConfiguration.ExtractPidFromGuid(process, this.Summary);
DryadProcessIdentifier identifier = new DryadProcessIdentifier(pid);
vi.SetStartInformation(this, machine, timeStamp, identifier, process);
}
break;
case "completed":
{
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
vi.SetState(ExecutedVertexInstance.VertexState.Successful);
vi.End = timeStamp;
vi.ExitCode = "";
this.UsefulCPUTime += vi.RunningTime;
break;
}
case "failed":
{
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
if (vi.State != ExecutedVertexInstance.VertexState.Started)
vi.SetState(ExecutedVertexInstance.VertexState.Cancelled);
else
{
vi.SetState(ExecutedVertexInstance.VertexState.Failed);
if (vi.RunningTime > TimeSpan.Zero)
this.WastedCPUTime += vi.RunningTime;
}
if (kvp.ContainsKey("errorstring"))
vi.AddErrorString(kvp["errorstring"]);
string exitcode;
if (kvp.TryGetValue("errorcode", out exitcode))
vi.ExitCode = exitcode;
vi.End = timeStamp;
break;
}
}
}
else if (kvp.ContainsKey("outputChannel"))
{
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
if (kvp.ContainsKey("errorstring"))
vi.AddErrorString(kvp["errorstring"]);
}
else if (kvp.ContainsKey("inputChannel"))
{
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
if (kvp.ContainsKey("errorstring"))
vi.AddErrorString(kvp["errorstring"]);
}
else if (kvp.ContainsKey("io"))
{
if (kvp["io"] == "starting")
{
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
int numberOfInputs = (int) TryGetNumeric(kvp, "numberOfInputs");
int numberOfOutputs = (int)TryGetNumeric(kvp, "numberOfOutputs");
if (vi.InputChannels == null)
vi.InputChannels = new Dictionary<int, ChannelEndpointDescription>();
for (int i = 0; i < numberOfInputs; i++)
{
string uri;
if (kvp.TryGetValue("uriIn." + i, out uri))
{
var ched = new ChannelEndpointDescription(false, i, uri, 0);
vi.InputChannels[i] = ched;
}
}
if (vi.OutputChannels == null)
vi.OutputChannels = new Dictionary<int, ChannelEndpointDescription>();
for (int i = 0; i < numberOfOutputs; i++)
{
string uri;
if (kvp.TryGetValue("uriOut." + i, out uri))
{
var ched = new ChannelEndpointDescription(false, i, uri, 0);
vi.OutputChannels[i] = ched;
}
}
}
else if (kvp["io"] == "total")
{
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
long totalRead = TryGetNumeric(kvp, "totalRead");
long tempRead = TryGetNumeric(kvp, "tempRead");
long tempReadInRack = TryGetNumeric(kvp, "tempReadInRack");
long tempReadCrossRack = TryGetNumeric(kvp, "tempReadCrossRack");
long localRead = TryGetNumeric(kvp, "localRead");
long totalWritten = TryGetNumeric(kvp, "totalWritten");
vi.DataRead = totalRead;
vi.DataWritten = totalWritten;
if (vi.InputChannels != null)
{
foreach (int ch in vi.InputChannels.Keys)
{
long bytes = TryGetNumeric(kvp, "rb." + ch);
vi.InputChannels[ch].Size = bytes;
}
}
if (vi.OutputChannels != null)
{
foreach (int ch in vi.OutputChannels.Keys)
{
long bytes = TryGetNumeric(kvp, "wb." + ch);
vi.OutputChannels[ch].Size = bytes;
}
}
this.TotalDataRead += totalRead;
this.LocalReadData += localRead;
this.CrossPodDataRead += tempReadCrossRack;
this.IntraPodDataRead += tempReadInRack;
}
else if (kvp["io"] == "running")
{
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
if (vi.InputChannels != null)
{
foreach (int ch in vi.InputChannels.Keys)
{
long bytes = TryGetNumeric(kvp, "rb." + ch);
vi.InputChannels[ch].Size = bytes;
bytes = TryGetNumeric(kvp, "tb." + ch);
vi.InputChannels[ch].TotalSize = bytes;
}
}
if (vi.InputChannels != null)
{
foreach (int ch in vi.OutputChannels.Keys)
{
long bytes = TryGetNumeric(kvp, "wb." + ch);
vi.OutputChannels[ch].Size = bytes;
}
}
long totalRead = TryGetNumeric(kvp, "totalRead");
long totalWritten = TryGetNumeric(kvp, "totalWritten");
vi.DataRead = totalRead;
vi.DataWritten = totalWritten;
}
}
}
return true;
}
/// <summary>
/// Parse a part of the 'originalInfo.txt' file to discover a set of channel endpoints.
/// </summary>
/// <param name="sr">Stream reader which contains the channel information.</param>
/// <returns>The list of channels, or null on failure.</returns>
/// <param name="uriprefix">If the channel is an output, prefix the path with this; this is null for inputs.</param>
/// <param name="skip">If true, do not return anything (still useful to advance the stream reader).</param>
/// <param name="fast">If true the channel sizes are not discovered; this is much faster, since no remote machines are queried for files.</param>
/// <param name="manager">Communication manager.</param>
private Dictionary<int, ChannelEndpointDescription> DiscoverOriginalInfoChannels(ISharedStreamReader sr, string uriprefix, bool skip, bool fast, CommManager manager)
{
bool isInput = uriprefix == null;
string countline = sr.ReadLine();
if (countline == null)
return null;
int channelCount;
int spaceIndex = countline.IndexOf(' ');
if (spaceIndex > 0)
countline = countline.Substring(0, spaceIndex);
bool success = int.TryParse(countline, out channelCount);
if (!success)
return null;
var channels = new Dictionary<int, ChannelEndpointDescription>(channelCount);
for (int i = 0; i < channelCount; i++)
{
string channel = sr.ReadLine();
if (channel == null)
{
manager.Progress(100);
return null;
}
if (!skip)
{
ChannelEndpointDescription desc = new ChannelEndpointDescription(isInput, i, channel, uriprefix, fast, manager.Status);
channels.Add(i, desc);
manager.Progress(i * 100 / channelCount);
}
}
manager.Progress(100);
if (skip)
return null;
return channels;
}