/// <exception cref="System.IO.IOException"/> public virtual void ReadFields(DataInput @in) { lock (this) { this.jobid = new JobID(); this.jobid.ReadFields(@in); this.setupProgress = @in.ReadFloat(); this.mapProgress = @in.ReadFloat(); this.reduceProgress = @in.ReadFloat(); this.cleanupProgress = @in.ReadFloat(); this.runState = WritableUtils.ReadEnum <JobStatus.State>(@in); this.startTime = @in.ReadLong(); this.user = StringInterner.WeakIntern(Text.ReadString(@in)); this.priority = WritableUtils.ReadEnum <JobPriority>(@in); this.schedulingInfo = StringInterner.WeakIntern(Text.ReadString(@in)); this.finishTime = @in.ReadLong(); this.isRetired = @in.ReadBoolean(); this.historyFile = StringInterner.WeakIntern(Text.ReadString(@in)); this.jobName = StringInterner.WeakIntern(Text.ReadString(@in)); this.trackingUrl = StringInterner.WeakIntern(Text.ReadString(@in)); this.jobFile = StringInterner.WeakIntern(Text.ReadString(@in)); this.isUber = @in.ReadBoolean(); // De-serialize the job's ACLs int numACLs = @in.ReadInt(); for (int i = 0; i < numACLs; i++) { JobACL aclType = WritableUtils.ReadEnum <JobACL>(@in); AccessControlList acl = new AccessControlList(" "); acl.ReadFields(@in); this.jobACLs[aclType] = acl; } } }
/// <exception cref="System.IO.IOException"/> public virtual void ReadFields(DataInput @in) { this.taskid.ReadFields(@in); this.progress = @in.ReadFloat(); this.state = StringInterner.WeakIntern(Text.ReadString(@in)); this.startTime = @in.ReadLong(); this.finishTime = @in.ReadLong(); diagnostics = WritableUtils.ReadStringArray(@in); counters = new Counters(); counters.ReadFields(@in); currentStatus = WritableUtils.ReadEnum <TIPStatus>(@in); if (currentStatus == TIPStatus.Running) { int num = WritableUtils.ReadVInt(@in); for (int i = 0; i < num; i++) { TaskAttemptID t = new TaskAttemptID(); t.ReadFields(@in); runningAttempts.AddItem(t); } } else { if (currentStatus == TIPStatus.Complete) { successfulAttempt.ReadFields(@in); } } }
/// <exception cref="System.IO.IOException"/> public virtual void ReadFields(DataInput @in) { numActiveTrackers = @in.ReadInt(); int numTrackerNames = @in.ReadInt(); if (numTrackerNames > 0) { for (int i = 0; i < numTrackerNames; i++) { string name = StringInterner.WeakIntern(Org.Apache.Hadoop.IO.Text.ReadString(@in) ); activeTrackers.AddItem(name); } } numBlacklistedTrackers = @in.ReadInt(); int blackListTrackerInfoSize = @in.ReadInt(); if (blackListTrackerInfoSize > 0) { for (int i = 0; i < blackListTrackerInfoSize; i++) { ClusterStatus.BlackListInfo info = new ClusterStatus.BlackListInfo(); info.ReadFields(@in); blacklistedTrackersInfo.AddItem(info); } } numExcludedNodes = @in.ReadInt(); ttExpiryInterval = @in.ReadLong(); map_tasks = @in.ReadInt(); reduce_tasks = @in.ReadInt(); max_map_tasks = @in.ReadInt(); max_reduce_tasks = @in.ReadInt(); status = WritableUtils.ReadEnum <Cluster.JobTrackerStatus>(@in); grayListedTrackers = @in.ReadInt(); }
private void HandleJobFailedEvent(JobUnsuccessfulCompletionEvent @event) { info.finishTime = @event.GetFinishTime(); info.finishedMaps = @event.GetFinishedMaps(); info.finishedReduces = @event.GetFinishedReduces(); info.jobStatus = StringInterner.WeakIntern(@event.GetStatus()); info.errorInfo = StringInterner.WeakIntern(@event.GetDiagnostics()); }
private void HandleTaskFailedEvent(TaskFailedEvent @event) { JobHistoryParser.TaskInfo taskInfo = info.tasksMap[@event.GetTaskId()]; taskInfo.status = TaskStatus.State.Failed.ToString(); taskInfo.finishTime = @event.GetFinishTime(); taskInfo.error = StringInterner.WeakIntern(@event.GetError()); taskInfo.failedDueToAttemptId = @event.GetFailedAttemptID(); taskInfo.counters = @event.GetCounters(); }
/// <exception cref="System.IO.IOException"/> public virtual void ReadFields(DataInput @in) { jobid.ReadFields(@in); this.jobFile = StringInterner.WeakIntern(Text.ReadString(@in)); this.url = StringInterner.WeakIntern(Text.ReadString(@in)); this.user = StringInterner.WeakIntern(Text.ReadString(@in)); this.name = StringInterner.WeakIntern(Text.ReadString(@in)); this.queueName = StringInterner.WeakIntern(Text.ReadString(@in)); }
private void HandleJobSubmittedEvent(JobSubmittedEvent @event) { info.jobid = @event.GetJobId(); info.jobname = @event.GetJobName(); info.username = StringInterner.WeakIntern(@event.GetUserName()); info.submitTime = @event.GetSubmitTime(); info.jobConfPath = @event.GetJobConfPath(); info.jobACLs = @event.GetJobAcls(); info.jobQueueName = StringInterner.WeakIntern(@event.GetJobQueueName()); }
public ContainerRemoteLaunchEvent(TaskAttemptId taskAttemptID, ContainerLaunchContext containerLaunchContext, Container allocatedContainer, Task remoteTask) : base(taskAttemptID, allocatedContainer.GetId(), StringInterner.WeakIntern(allocatedContainer .GetNodeId().ToString()), allocatedContainer.GetContainerToken(), ContainerLauncher.EventType .ContainerRemoteLaunch) { this.allocatedContainer = allocatedContainer; this.containerLaunchContext = containerLaunchContext; this.task = remoteTask; }
/// <exception cref="System.IO.IOException"/> public override void ReadFields(DataInput @in) { lock (this) { name = StringInterner.WeakIntern(Text.ReadString(@in)); displayName = @in.ReadBoolean() ? StringInterner.WeakIntern(Text.ReadString(@in)) : name; value = WritableUtils.ReadVLong(@in); } }
public virtual void TestWeakIntern() { string weakInternLiteralABC = StringInterner.WeakIntern("ABC"); string weakInternSubstringABC = StringInterner.WeakIntern(Runtime.Substring ("ABCDE", 0, 3)); string weakInternHeapABC = StringInterner.WeakIntern(new string("ABC")); NUnit.Framework.Assert.AreSame(weakInternLiteralABC, weakInternSubstringABC); NUnit.Framework.Assert.AreSame(weakInternLiteralABC, weakInternHeapABC); NUnit.Framework.Assert.AreSame(weakInternSubstringABC, weakInternHeapABC); }
private void HandleTaskAttemptFinishedEvent(TaskAttemptFinishedEvent @event) { JobHistoryParser.TaskInfo taskInfo = info.tasksMap[@event.GetTaskId()]; JobHistoryParser.TaskAttemptInfo attemptInfo = taskInfo.attemptsMap[@event.GetAttemptId ()]; attemptInfo.finishTime = @event.GetFinishTime(); attemptInfo.status = StringInterner.WeakIntern(@event.GetTaskStatus()); attemptInfo.state = StringInterner.WeakIntern(@event.GetState()); attemptInfo.counters = @event.GetCounters(); attemptInfo.hostname = StringInterner.WeakIntern(@event.GetHostname()); info.completedTaskAttemptsMap[@event.GetAttemptId()] = attemptInfo; }
/// <exception cref="System.IO.IOException"/> public virtual void ReadFields(DataInput @in) { lock (this) { int version = WritableUtils.ReadVInt(@in); if (version != groupFactory.Version()) { throw new IOException("Counters version mismatch, expected " + groupFactory.Version () + " got " + version); } int numFGroups = WritableUtils.ReadVInt(@in); fgroups.Clear(); AbstractCounters.GroupType[] groupTypes = AbstractCounters.GroupType.Values(); while (numFGroups-- > 0) { AbstractCounters.GroupType groupType = groupTypes[WritableUtils.ReadVInt(@in)]; G group; switch (groupType) { case AbstractCounters.GroupType.Filesystem: { // with nothing group = groupFactory.NewFileSystemGroup(); break; } case AbstractCounters.GroupType.Framework: { // with group id group = groupFactory.NewFrameworkGroup(WritableUtils.ReadVInt(@in)); break; } default: { // Silence dumb compiler, as it would've thrown earlier throw new IOException("Unexpected counter group type: " + groupType); } } group.ReadFields(@in); fgroups[group.GetName()] = group; } int numGroups = WritableUtils.ReadVInt(@in); while (numGroups-- > 0) { limits.CheckGroups(groups.Count + 1); G group = groupFactory.NewGenericGroup(StringInterner.WeakIntern(Text.ReadString( @in)), null, limits); group.ReadFields(@in); groups[group.GetName()] = group; } } }
/// <exception cref="System.IO.IOException"/> private Type ReadClass(DataInput @in) { string className = StringInterner.WeakIntern(Text.ReadString(@in)); try { return(conf.GetClassByName(className)); } catch (TypeLoadException e) { throw new RuntimeException("readObject can't find class", e); } }
// found nothing /// <summary>Parse a pre 0.21 counters string into a counter object.</summary> /// <?/> /// <?/> /// <?/> /// <param name="compactString">to parse</param> /// <param name="counters">an empty counters object to hold the result</param> /// <returns>the counters object holding the result</returns> /// <exception cref="Sharpen.ParseException"/> public static T ParseEscapedCompactString <C, G, T>(string compactString, T counters ) where C : Counter where G : CounterGroupBase <C> where T : AbstractCounters <C, G> { IntWritable index = new IntWritable(0); // Get the group to work on string groupString = GetBlock(compactString, GroupOpen, GroupClose, index); while (groupString != null) { IntWritable groupIndex = new IntWritable(0); // Get the actual name string groupName = StringInterner.WeakIntern(GetBlock(groupString, UnitOpen, UnitClose , groupIndex)); groupName = StringInterner.WeakIntern(Unescape(groupName)); // Get the display name string groupDisplayName = StringInterner.WeakIntern(GetBlock(groupString, UnitOpen , UnitClose, groupIndex)); groupDisplayName = StringInterner.WeakIntern(Unescape(groupDisplayName)); // Get the counters G group = counters.GetGroup(groupName); group.SetDisplayName(groupDisplayName); string counterString = GetBlock(groupString, CounterOpen, CounterClose, groupIndex ); while (counterString != null) { IntWritable counterIndex = new IntWritable(0); // Get the actual name string counterName = StringInterner.WeakIntern(GetBlock(counterString, UnitOpen, UnitClose, counterIndex)); counterName = StringInterner.WeakIntern(Unescape(counterName)); // Get the display name string counterDisplayName = StringInterner.WeakIntern(GetBlock(counterString, UnitOpen , UnitClose, counterIndex)); counterDisplayName = StringInterner.WeakIntern(Unescape(counterDisplayName)); // Get the value long value = long.Parse(GetBlock(counterString, UnitOpen, UnitClose, counterIndex )); // Add the counter Counter counter = group.FindCounter(counterName); counter.SetDisplayName(counterDisplayName); counter.Increment(value); // Get the next counter counterString = GetBlock(groupString, CounterOpen, CounterClose, groupIndex); } groupString = GetBlock(compactString, GroupOpen, GroupClose, index); } return(counters); }
public static void AddToEnvironment(IDictionary <string, string> environment, string variable, string value, string classPathSeparator) { string val = environment[variable]; if (val == null) { val = value; } else { val = val + classPathSeparator + value; } environment[StringInterner.WeakIntern(variable)] = StringInterner.WeakIntern(val); }
/// <exception cref="System.IO.IOException"/> public virtual void ReadFields(DataInput @in) { lock (this) { displayName = StringInterner.WeakIntern(Text.ReadString(@in)); counters.Clear(); int size = WritableUtils.ReadVInt(@in); for (int i = 0; i < size; i++) { T counter = NewCounter(); counter.ReadFields(@in); counters[counter.GetName()] = counter; limits.IncrCounters(); } } }
private void HandleTaskAttemptStartedEvent(TaskAttemptStartedEvent @event) { TaskAttemptID attemptId = @event.GetTaskAttemptId(); JobHistoryParser.TaskInfo taskInfo = info.tasksMap[@event.GetTaskId()]; JobHistoryParser.TaskAttemptInfo attemptInfo = new JobHistoryParser.TaskAttemptInfo (); attemptInfo.startTime = @event.GetStartTime(); attemptInfo.attemptId = @event.GetTaskAttemptId(); attemptInfo.httpPort = @event.GetHttpPort(); attemptInfo.trackerName = StringInterner.WeakIntern(@event.GetTrackerName()); attemptInfo.taskType = @event.GetTaskType(); attemptInfo.shufflePort = @event.GetShufflePort(); attemptInfo.containerId = @event.GetContainerId(); taskInfo.attemptsMap[attemptId] = attemptInfo; }
private void HandleAMStartedEvent(AMStartedEvent @event) { JobHistoryParser.AMInfo amInfo = new JobHistoryParser.AMInfo(); amInfo.appAttemptId = @event.GetAppAttemptId(); amInfo.startTime = @event.GetStartTime(); amInfo.containerId = @event.GetContainerId(); amInfo.nodeManagerHost = StringInterner.WeakIntern(@event.GetNodeManagerHost()); amInfo.nodeManagerPort = @event.GetNodeManagerPort(); amInfo.nodeManagerHttpPort = @event.GetNodeManagerHttpPort(); if (info.amInfos == null) { info.amInfos = new List <JobHistoryParser.AMInfo>(); } info.amInfos.AddItem(amInfo); info.latestAmInfo = amInfo; }
/// <exception cref="System.IO.IOException"/> public virtual void ReadFields(DataInput @in) { this.taskid.ReadFields(@in); SetProgress(@in.ReadFloat()); this.numSlots = @in.ReadInt(); this.runState = WritableUtils.ReadEnum <TaskStatus.State>(@in); SetDiagnosticInfo(StringInterner.WeakIntern(Text.ReadString(@in))); SetStateString(StringInterner.WeakIntern(Text.ReadString(@in))); this.phase = WritableUtils.ReadEnum <TaskStatus.Phase>(@in); this.startTime = @in.ReadLong(); this.finishTime = @in.ReadLong(); counters = new Counters(); this.includeAllCounters = @in.ReadBoolean(); this.outputSize = @in.ReadLong(); counters.ReadFields(@in); nextRecordRange.ReadFields(@in); }
/// <exception cref="System.IO.IOException"/> public virtual void ReportDiagnosticInfo(TaskAttemptID taskAttemptID, string diagnosticInfo ) { diagnosticInfo = StringInterner.WeakIntern(diagnosticInfo); Log.Info("Diagnostics report from " + taskAttemptID.ToString() + ": " + diagnosticInfo ); TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID); taskHeartbeatHandler.Progressing(attemptID); // This is mainly used for cases where we want to propagate exception traces // of tasks that fail. // This call exists as a hadoop mapreduce legacy wherein all changes in // counters/progress/phase/output-size are reported through statusUpdate() // call but not diagnosticInformation. context.GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attemptID, diagnosticInfo)); }
private void HandleTaskAttemptFailedEvent(TaskAttemptUnsuccessfulCompletionEvent @event) { JobHistoryParser.TaskInfo taskInfo = info.tasksMap[@event.GetTaskId()]; if (taskInfo == null) { Log.Warn("TaskInfo is null for TaskAttemptUnsuccessfulCompletionEvent" + " taskId: " + @event.GetTaskId().ToString()); return; } JobHistoryParser.TaskAttemptInfo attemptInfo = taskInfo.attemptsMap[@event.GetTaskAttemptId ()]; if (attemptInfo == null) { Log.Warn("AttemptInfo is null for TaskAttemptUnsuccessfulCompletionEvent" + " taskAttemptId: " + @event.GetTaskAttemptId().ToString()); return; } attemptInfo.finishTime = @event.GetFinishTime(); attemptInfo.error = StringInterner.WeakIntern(@event.GetError()); attemptInfo.status = StringInterner.WeakIntern(@event.GetTaskStatus()); attemptInfo.hostname = StringInterner.WeakIntern(@event.GetHostname()); attemptInfo.port = @event.GetPort(); attemptInfo.rackname = StringInterner.WeakIntern(@event.GetRackName()); attemptInfo.shuffleFinishTime = @event.GetFinishTime(); attemptInfo.sortFinishTime = @event.GetFinishTime(); attemptInfo.mapFinishTime = @event.GetFinishTime(); attemptInfo.counters = @event.GetCounters(); if (TaskStatus.State.Succeeded.ToString().Equals(taskInfo.status)) { //this is a successful task if (attemptInfo.GetAttemptId().Equals(taskInfo.GetSuccessfulAttemptId())) { // the failed attempt is the one that made this task successful // so its no longer successful. Reset fields set in // handleTaskFinishedEvent() taskInfo.counters = null; taskInfo.finishTime = -1; taskInfo.status = null; taskInfo.successfulAttemptId = null; } } info.completedTaskAttemptsMap[@event.GetTaskAttemptId()] = attemptInfo; }
internal static Counters FromAvro(JhCounters counters) { Counters result = new Counters(); if (counters != null) { foreach (JhCounterGroup g in counters.groups) { CounterGroup group = result.AddGroup(StringInterner.WeakIntern(g.name.ToString()) , StringInterner.WeakIntern(g.displayName.ToString())); foreach (JhCounter c in g.counts) { group.AddCounter(StringInterner.WeakIntern(c.name.ToString()), StringInterner.WeakIntern (c.displayName.ToString()), c.value); } } } return(result); }
/// <exception cref="System.IO.IOException"/> public virtual void ReadFields(DataInput @in) { queueName = StringInterner.WeakIntern(Text.ReadString(@in)); queueState = WritableUtils.ReadEnum <QueueState>(@in); schedulingInfo = StringInterner.WeakIntern(Text.ReadString(@in)); int length = @in.ReadInt(); stats = new JobStatus[length]; for (int i = 0; i < length; i++) { stats[i] = new JobStatus(); stats[i].ReadFields(@in); } int count = @in.ReadInt(); children.Clear(); for (int i_1 = 0; i_1 < count; i_1++) { Org.Apache.Hadoop.Mapreduce.QueueInfo childQueueInfo = new Org.Apache.Hadoop.Mapreduce.QueueInfo (); childQueueInfo.ReadFields(@in); children.AddItem(childQueueInfo); } }
/// <exception cref="System.IO.IOException"/> public virtual void ReadFields(DataInput @in) { queueName = StringInterner.WeakIntern(Text.ReadString(@in)); operations = WritableUtils.ReadStringArray(@in); }
/// <exception cref="System.IO.IOException"/> public virtual void ReadFields(DataInput @in) { trackerName = StringInterner.WeakIntern(Text.ReadString(@in)); reasonForBlackListing = StringInterner.WeakIntern(Text.ReadString(@in)); blackListReport = StringInterner.WeakIntern(Text.ReadString(@in)); }
private string Intern(string key) { return(StringInterner.WeakIntern(key)); }