/// <summary> /// Discover the vertex channels in a Scope-generated vcmdStart*xml file. /// </summary> /// <returns>True if the discovery was successful.</returns> /// <param name="inputs">If true discover the inputs.</param> /// <param name="outputs">If true discover the outputs.</param> /// <param name="fast">If true do not discover the channel sizes (much faster).</param> /// <param name="manager">Communication manager.</param> // ReSharper disable UnusedParameter.Global public bool DiscoverScopeChannels(bool inputs, bool outputs, bool fast, CommManager manager) // ReSharper restore UnusedParameter.Global { // find the xml file var files = this.WorkDirectory.GetFilesAndFolders("vcmdStart*.xml").ToList(); if (files.Count != 1) { manager.Status("Cannot locate vcmdStart*.xml file", StatusKind.Error); return false; } ISharedStreamReader sr = files.First().GetStream(); if (sr.Exception != null) { manager.Status("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error); return false; } // ReSharper disable PossibleNullReferenceException XDocument plan = XDocument.Parse(sr.ReadToEnd(manager.Token)); if (inputs && this.InputChannels == null) { var channels = new Dictionary<int, ChannelEndpointDescription>(); IEnumerable<XElement> inputsData = plan.Root.Element("inputs").Elements(); int chno = 0; foreach (var e in inputsData) { string chpath = e.Attribute("path").Value; long size = long.Parse(e.Attribute("length").Value); ChannelEndpointDescription desc = new ChannelEndpointDescription(true, chno, chpath, size); channels.Add(chno, desc); chno++; } this.InputChannels = channels; } if (outputs && this.OutputChannels == null) { var channels = new Dictionary<int, ChannelEndpointDescription>(); IEnumerable<XElement> inputsData = plan.Root.Element("outputs").Elements(); int chno = 0; foreach (var e in inputsData) { string chpath = e.Attribute("path").Value; ChannelEndpointDescription desc = new ChannelEndpointDescription(true, chno, chpath, -1); channels.Add(chno, desc); chno ++; } this.OutputChannels = channels; } // ReSharper restore PossibleNullReferenceException sr.Close(); return true; }
/// <summary> /// New JM stdout parsing code, for YARN-based DryadLINQ. /// </summary> /// <param name="line">Line to parse.</param> /// <returns>False if the line terminated in a quoted string and has to be combined with the next line.</returns> private bool ParseStdoutLineNew(string line) { if (string.IsNullOrWhiteSpace(line)) return true; Dictionary<string, string> kvp = Utilities.ParseCSVKVP(line); if (kvp == null) return false; var strTs = kvp["logtimelocal"]; int cutOff = strTs.IndexOf("UTC"); if (cutOff >= 0) { strTs = strTs.Substring(0, cutOff); } DateTime timeStamp = DateTime.Parse(strTs, CultureInfo.InvariantCulture); timeStamp = timeStamp.ToLocalTime(); this.lastTimestampSeen = timeStamp; if (kvp.ContainsKey("job")) { string operation = kvp["job"]; switch (operation) { case "start": this.ManagerVertex.SetStartInformation(this, this.Summary.Machine, timeStamp, this.Summary.ManagerProcessGuid, ""); this.ManagerVertex.StartCommandTime = this.ManagerVertex.CreationTime = this.ManagerVertex.VertexScheduleTime = timeStamp; break; case "stop": this.ManagerVertex.End = timeStamp; string exitcode; if (kvp.TryGetValue("exitcode", out exitcode)) { this.ErrorCode = exitcode; int numCode = Convert.ToInt32(exitcode, 16); if (numCode == 0) { this.ManagerVertex.SetState(ExecutedVertexInstance.VertexState.Successful); } else { this.ManagerVertex.SetState(ExecutedVertexInstance.VertexState.Failed); } } string errorstring; if (kvp.TryGetValue("errorstring", out errorstring)) { this.ManagerVertex.AddErrorString(errorstring); this.AbortingMsg = errorstring; } break; } } else if (kvp.ContainsKey("vertex")) { string vertex = kvp["vertex"]; int number; int version; int dot = vertex.IndexOf('.'); if (dot < 0) { number = int.Parse(vertex); version = int.Parse(kvp["version"]); } else { number = int.Parse(vertex.Substring(0, dot)); version = int.Parse(vertex.Substring(dot + 1)); } if (kvp.ContainsKey("transition")) { string transition = kvp["transition"]; switch (transition) { case "created": { string name = kvp["name"]; ExecutedVertexInstance vi = new ExecutedVertexInstance(this, number, version, name, "", timeStamp); this.jobVertices.Add(vi); } break; case "starting": { // not doing anything break; } case "running": { string process; kvp.TryGetValue("id", out process); if (process == null) kvp.TryGetValue("process", out process); string machine = kvp["computer"]; ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version); this.jobVertices.Remap(vi, process); string pid = this.ClusterConfiguration.ExtractPidFromGuid(process, this.Summary); DryadProcessIdentifier identifier = new DryadProcessIdentifier(pid); vi.SetStartInformation(this, machine, timeStamp, identifier, process); } break; case "completed": { ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version); vi.SetState(ExecutedVertexInstance.VertexState.Successful); vi.End = timeStamp; vi.ExitCode = ""; this.UsefulCPUTime += vi.RunningTime; break; } case "failed": { ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version); if (vi.State != ExecutedVertexInstance.VertexState.Started) vi.SetState(ExecutedVertexInstance.VertexState.Cancelled); else { vi.SetState(ExecutedVertexInstance.VertexState.Failed); if (vi.RunningTime > TimeSpan.Zero) this.WastedCPUTime += vi.RunningTime; } if (kvp.ContainsKey("errorstring")) vi.AddErrorString(kvp["errorstring"]); string exitcode; if (kvp.TryGetValue("errorcode", out exitcode)) vi.ExitCode = exitcode; vi.End = timeStamp; break; } } } else if (kvp.ContainsKey("outputChannel")) { ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version); if (kvp.ContainsKey("errorstring")) vi.AddErrorString(kvp["errorstring"]); } else if (kvp.ContainsKey("inputChannel")) { ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version); if (kvp.ContainsKey("errorstring")) vi.AddErrorString(kvp["errorstring"]); } else if (kvp.ContainsKey("io")) { if (kvp["io"] == "starting") { ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version); int numberOfInputs = (int) TryGetNumeric(kvp, "numberOfInputs"); int numberOfOutputs = (int)TryGetNumeric(kvp, "numberOfOutputs"); if (vi.InputChannels == null) vi.InputChannels = new Dictionary<int, ChannelEndpointDescription>(); for (int i = 0; i < numberOfInputs; i++) { string uri; if (kvp.TryGetValue("uriIn." + i, out uri)) { var ched = new ChannelEndpointDescription(false, i, uri, 0); vi.InputChannels[i] = ched; } } if (vi.OutputChannels == null) vi.OutputChannels = new Dictionary<int, ChannelEndpointDescription>(); for (int i = 0; i < numberOfOutputs; i++) { string uri; if (kvp.TryGetValue("uriOut." + i, out uri)) { var ched = new ChannelEndpointDescription(false, i, uri, 0); vi.OutputChannels[i] = ched; } } } else if (kvp["io"] == "total") { ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version); long totalRead = TryGetNumeric(kvp, "totalRead"); long tempRead = TryGetNumeric(kvp, "tempRead"); long tempReadInRack = TryGetNumeric(kvp, "tempReadInRack"); long tempReadCrossRack = TryGetNumeric(kvp, "tempReadCrossRack"); long localRead = TryGetNumeric(kvp, "localRead"); long totalWritten = TryGetNumeric(kvp, "totalWritten"); vi.DataRead = totalRead; vi.DataWritten = totalWritten; if (vi.InputChannels != null) { foreach (int ch in vi.InputChannels.Keys) { long bytes = TryGetNumeric(kvp, "rb." + ch); vi.InputChannels[ch].Size = bytes; } } if (vi.OutputChannels != null) { foreach (int ch in vi.OutputChannels.Keys) { long bytes = TryGetNumeric(kvp, "wb." + ch); vi.OutputChannels[ch].Size = bytes; } } this.TotalDataRead += totalRead; this.LocalReadData += localRead; this.CrossPodDataRead += tempReadCrossRack; this.IntraPodDataRead += tempReadInRack; } else if (kvp["io"] == "running") { ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version); if (vi.InputChannels != null) { foreach (int ch in vi.InputChannels.Keys) { long bytes = TryGetNumeric(kvp, "rb." + ch); vi.InputChannels[ch].Size = bytes; bytes = TryGetNumeric(kvp, "tb." + ch); vi.InputChannels[ch].TotalSize = bytes; } } if (vi.InputChannels != null) { foreach (int ch in vi.OutputChannels.Keys) { long bytes = TryGetNumeric(kvp, "wb." + ch); vi.OutputChannels[ch].Size = bytes; } } long totalRead = TryGetNumeric(kvp, "totalRead"); long totalWritten = TryGetNumeric(kvp, "totalWritten"); vi.DataRead = totalRead; vi.DataWritten = totalWritten; } } } return true; }
/// <summary> /// Parse a part of the 'originalInfo.txt' file to discover a set of channel endpoints. /// </summary> /// <param name="sr">Stream reader which contains the channel information.</param> /// <returns>The list of channels, or null on failure.</returns> /// <param name="uriprefix">If the channel is an output, prefix the path with this; this is null for inputs.</param> /// <param name="skip">If true, do not return anything (still useful to advance the stream reader).</param> /// <param name="fast">If true the channel sizes are not discovered; this is much faster, since no remote machines are queried for files.</param> /// <param name="manager">Communication manager.</param> private Dictionary<int, ChannelEndpointDescription> DiscoverOriginalInfoChannels(ISharedStreamReader sr, string uriprefix, bool skip, bool fast, CommManager manager) { bool isInput = uriprefix == null; string countline = sr.ReadLine(); if (countline == null) return null; int channelCount; int spaceIndex = countline.IndexOf(' '); if (spaceIndex > 0) countline = countline.Substring(0, spaceIndex); bool success = int.TryParse(countline, out channelCount); if (!success) return null; var channels = new Dictionary<int, ChannelEndpointDescription>(channelCount); for (int i = 0; i < channelCount; i++) { string channel = sr.ReadLine(); if (channel == null) { manager.Progress(100); return null; } if (!skip) { ChannelEndpointDescription desc = new ChannelEndpointDescription(isInput, i, channel, uriprefix, fast, manager.Status); channels.Add(i, desc); manager.Progress(i * 100 / channelCount); } } manager.Progress(100); if (skip) return null; return channels; }