/// <summary>Absorbs one TaskAttemptStatus</summary>
/// <param name="reportedStatus">
/// the status report that we got from a task attempt
/// that we want to fold into the speculation data for this job
/// </param>
/// <param name="timestamp">
/// the time this status corresponds to. This matters
/// because statuses contain progress.
/// </param>
protected internal virtual void StatusUpdate(TaskAttemptStatusUpdateEvent.TaskAttemptStatus
reportedStatus, long timestamp)
{
string stateString = reportedStatus.taskState.ToString();
TaskAttemptId attemptID = reportedStatus.id;
TaskId taskID = attemptID.GetTaskId();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(taskID.GetJobId()
);
if (job == null)
{
return;
}
Task task = job.GetTask(taskID);
if (task == null)
{
return;
}
estimator.UpdateAttempt(reportedStatus, timestamp);
if (stateString.Equals(TaskAttemptState.Running.ToString()))
{
runningTasks.PutIfAbsent(taskID, true);
}
else
{
runningTasks.Remove(taskID, true);
if (!stateString.Equals(TaskAttemptState.Starting.ToString()))
{
Sharpen.Collections.Remove(runningTaskAttemptStatistics, attemptID);
}
}
}
private ConcurrentMap <ApplicationAttemptId, ApplicationAttemptHistoryData> GetSubMap
(ApplicationId appId)
{
applicationAttemptData.PutIfAbsent(appId, new ConcurrentHashMap <ApplicationAttemptId
, ApplicationAttemptHistoryData>());
return(applicationAttemptData[appId]);
}
private ConcurrentMap <ContainerId, ContainerHistoryData> GetSubMap(ApplicationAttemptId
appAttemptId)
{
containerData.PutIfAbsent(appAttemptId, new ConcurrentHashMap <ContainerId, ContainerHistoryData
>());
return(containerData[appAttemptId]);
}
private void IncorporateReading(TaskAttemptId attemptID, float newProgress, long
newTime)
{
//TODO: Refactor this method, it seems more complicated than necessary.
AtomicReference <ExponentiallySmoothedTaskRuntimeEstimator.EstimateVector> vectorRef
= estimates[attemptID];
if (vectorRef == null)
{
estimates.PutIfAbsent(attemptID, new AtomicReference <ExponentiallySmoothedTaskRuntimeEstimator.EstimateVector
>(null));
IncorporateReading(attemptID, newProgress, newTime);
return;
}
ExponentiallySmoothedTaskRuntimeEstimator.EstimateVector oldVector = vectorRef.Get
();
if (oldVector == null)
{
if (vectorRef.CompareAndSet(null, new ExponentiallySmoothedTaskRuntimeEstimator.EstimateVector
(this, -1.0, 0.0F, long.MinValue)))
{
return;
}
IncorporateReading(attemptID, newProgress, newTime);
return;
}
while (!vectorRef.CompareAndSet(oldVector, oldVector.Incorporate(newProgress, newTime
)))
{
oldVector = vectorRef.Get();
}
}
/// <summary>
/// Lazily initializes the internal data structure for tracking the specified
/// phase and step.
/// </summary>
/// <remarks>
/// Lazily initializes the internal data structure for tracking the specified
/// phase and step. Returns either the newly initialized data structure or the
/// existing one. Initialization is atomic, so there is no risk of lost updates
/// even if multiple threads attempt to initialize the same step simultaneously.
/// </remarks>
/// <param name="phase">Phase to initialize</param>
/// <param name="step">Step to initialize</param>
/// <returns>StepTracking newly initialized, or existing if found</returns>
private StepTracking LazyInitStep(Phase phase, Step step)
{
ConcurrentMap <Step, StepTracking> steps = phases[phase].steps;
if (!steps.Contains(step))
{
steps.PutIfAbsent(step, new StepTracking());
}
return(steps[step]);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void AddStartingContainer(NMClientImpl.StartedContainer startedContainer)
{
if (startedContainers.PutIfAbsent(startedContainer.containerId, startedContainer)
!= null)
{
throw RPCUtil.GetRemoteException("Container " + startedContainer.containerId.ToString
() + " is already started");
}
startedContainers[startedContainer.GetContainerId()] = startedContainer;
}
/// <summary>
/// Registers the provider.
/// </summary>
/// <param name="provider"> the provider to register, not null </param>
/// <exception cref="ZoneRulesException"> if unable to complete the registration </exception>
private static void RegisterProvider0(ZoneRulesProvider provider)
{
foreach (String zoneId in provider.ProvideZoneIds())
{
Objects.RequireNonNull(zoneId, "zoneId");
ZoneRulesProvider old = ZONES.PutIfAbsent(zoneId, provider);
if (old != null)
{
throw new ZoneRulesException("Unable to register zone as one already registered with that ID: " + zoneId + ", currently loading from provider: " + provider);
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void ApplicationStarted(ApplicationStartData appStart)
{
ApplicationHistoryData oldData = applicationData.PutIfAbsent(appStart.GetApplicationId
(), ApplicationHistoryData.NewInstance(appStart.GetApplicationId(), appStart.GetApplicationName
(), appStart.GetApplicationType(), appStart.GetQueue(), appStart.GetUser(), appStart
.GetSubmitTime(), appStart.GetStartTime(), long.MaxValue, null, null, null));
if (oldData != null)
{
throw new IOException("The start information of application " + appStart.GetApplicationId
() + " is already stored.");
}
}
private string CheckScheme(string scheme)
{
string @fixed = StringUtils.ToUpperCase(scheme);
string interned = schemes.PutIfAbsent(@fixed, @fixed);
if (schemes.Count > MaxNumSchemes)
{
// mistakes or abuses
throw new ArgumentException("too many schemes? " + schemes.Count + " when process scheme: "
+ scheme);
}
return(interned == null ? @fixed : interned);
}
/// <summary>
/// Obtains the DecimalStyle for the specified locale.
/// <para>
/// This method provides access to locale sensitive decimal style symbols.
///
/// </para>
/// </summary>
/// <param name="locale"> the locale, not null </param>
/// <returns> the decimal style, not null </returns>
public static DecimalStyle Of(Locale locale)
{
Objects.RequireNonNull(locale, "locale");
DecimalStyle info = CACHE[locale];
if (info == null)
{
info = Create(locale);
CACHE.PutIfAbsent(locale, info);
info = CACHE[locale];
}
return(info);
}
private static Currency GetInstance(String currencyCode, int defaultFractionDigits, int numericCode)
{
// Try to look up the currency code in the instances table.
// This does the null pointer check as a side effect.
// Also, if there already is an entry, the currencyCode must be valid.
Currency instance = Instances[currencyCode];
if (instance != null)
{
return(instance);
}
if (defaultFractionDigits == Integer.MinValue)
{
// Currency code not internally generated, need to verify first
// A currency code must have 3 characters and exist in the main table
// or in the list of other currencies.
if (currencyCode.Length() != 3)
{
throw new IllegalArgumentException();
}
char char1 = currencyCode.CharAt(0);
char char2 = currencyCode.CharAt(1);
int tableEntry = GetMainTableEntry(char1, char2);
if ((tableEntry & COUNTRY_TYPE_MASK) == SIMPLE_CASE_COUNTRY_MASK && tableEntry != INVALID_COUNTRY_ENTRY && currencyCode.CharAt(2) - 'A' == (tableEntry & SIMPLE_CASE_COUNTRY_FINAL_CHAR_MASK))
{
defaultFractionDigits = (tableEntry & SIMPLE_CASE_COUNTRY_DEFAULT_DIGITS_MASK) >> SIMPLE_CASE_COUNTRY_DEFAULT_DIGITS_SHIFT;
numericCode = (tableEntry & NUMERIC_CODE_MASK) >> NUMERIC_CODE_SHIFT;
}
else
{
// Check for '-' separately so we don't get false hits in the table.
if (currencyCode.CharAt(2) == '-')
{
throw new IllegalArgumentException();
}
int index = OtherCurrencies.IndexOf(currencyCode);
if (index == -1)
{
throw new IllegalArgumentException();
}
defaultFractionDigits = OtherCurrenciesDFD[index / 4];
numericCode = OtherCurrenciesNumericCode[index / 4];
}
}
Currency currencyVal = new Currency(currencyCode, defaultFractionDigits, numericCode);
instance = Instances.PutIfAbsent(currencyCode, currencyVal);
return(instance != null ? instance : currencyVal);
}
/* ************************************************************* */
// This section contains the code that gets run for a SpeculatorEvent
private AtomicInteger ContainerNeed(TaskId taskID)
{
JobId jobID = taskID.GetJobId();
TaskType taskType = taskID.GetTaskType();
ConcurrentMap <JobId, AtomicInteger> relevantMap = taskType == TaskType.Map ? mapContainerNeeds
: reduceContainerNeeds;
AtomicInteger result = relevantMap[jobID];
if (result == null)
{
relevantMap.PutIfAbsent(jobID, new AtomicInteger(0));
result = relevantMap[jobID];
}
return(result);
}
/// <exception cref="System.IO.IOException"/>
public virtual void ContainerStarted(ContainerStartData containerStart)
{
ConcurrentMap <ContainerId, ContainerHistoryData> subMap = GetSubMap(containerStart
.GetContainerId().GetApplicationAttemptId());
ContainerHistoryData oldData = subMap.PutIfAbsent(containerStart.GetContainerId()
, ContainerHistoryData.NewInstance(containerStart.GetContainerId(), containerStart
.GetAllocatedResource(), containerStart.GetAssignedNode(), containerStart.GetPriority
(), containerStart.GetStartTime(), long.MaxValue, null, int.MaxValue, null));
if (oldData != null)
{
throw new IOException("The start information of container " + containerStart.GetContainerId
() + " is already stored.");
}
}
private void Add(ListenerHandle handle)
{
IList <ListenerHandle> list = lists.Get(handle.type);
if (list == null)
{
CopyOnWriteArrayList <ListenerHandle> newList;
newList = new CopyOnWriteArrayList <ListenerHandle>();
list = lists.PutIfAbsent(handle.type, newList);
if (list == null)
{
list = newList;
}
}
list.AddItem(handle);
}
/// <exception cref="System.IO.IOException"/>
public virtual void ApplicationAttemptStarted(ApplicationAttemptStartData appAttemptStart
)
{
ConcurrentMap <ApplicationAttemptId, ApplicationAttemptHistoryData> subMap = GetSubMap
(appAttemptStart.GetApplicationAttemptId().GetApplicationId());
ApplicationAttemptHistoryData oldData = subMap.PutIfAbsent(appAttemptStart.GetApplicationAttemptId
(), ApplicationAttemptHistoryData.NewInstance(appAttemptStart.GetApplicationAttemptId
(), appAttemptStart.GetHost(), appAttemptStart.GetRPCPort(), appAttemptStart.GetMasterContainerId
(), null, null, null, null));
if (oldData != null)
{
throw new IOException("The start information of application attempt " + appAttemptStart
.GetApplicationAttemptId() + " is already stored.");
}
}
public virtual object GetClient(Type protocol, long clientVersion, IPEndPoint addr
, Configuration conf)
{
Constructor <object> constructor = cache[protocol];
if (constructor == null)
{
Type pbClazz = null;
try
{
pbClazz = localConf.GetClassByName(GetPBImplClassName(protocol));
}
catch (TypeLoadException e)
{
throw new YarnRuntimeException("Failed to load class: [" + GetPBImplClassName(protocol
) + "]", e);
}
try
{
constructor = pbClazz.GetConstructor(typeof(long), typeof(IPEndPoint), typeof(Configuration
));
cache.PutIfAbsent(protocol, constructor);
}
catch (MissingMethodException e)
{
throw new YarnRuntimeException("Could not find constructor with params: " + typeof(
long) + ", " + typeof(IPEndPoint) + ", " + typeof(Configuration), e);
}
}
try
{
object retObject = constructor.NewInstance(clientVersion, addr, conf);
return(retObject);
}
catch (TargetInvocationException e)
{
throw new YarnRuntimeException(e);
}
catch (MemberAccessException e)
{
throw new YarnRuntimeException(e);
}
catch (InstantiationException e)
{
throw new YarnRuntimeException(e);
}
}
public virtual T NewRecordInstance <T>()
{
System.Type clazz = typeof(T);
Constructor <object> constructor = cache[clazz];
if (constructor == null)
{
Type pbClazz = null;
try
{
pbClazz = localConf.GetClassByName(GetPBImplClassName(clazz));
}
catch (TypeLoadException e)
{
throw new YarnRuntimeException("Failed to load class: [" + GetPBImplClassName(clazz
) + "]", e);
}
try
{
constructor = pbClazz.GetConstructor();
cache.PutIfAbsent(clazz, constructor);
}
catch (MissingMethodException e)
{
throw new YarnRuntimeException("Could not find 0 argument constructor", e);
}
}
try
{
object retObject = constructor.NewInstance();
return((T)retObject);
}
catch (TargetInvocationException e)
{
throw new YarnRuntimeException(e);
}
catch (MemberAccessException e)
{
throw new YarnRuntimeException(e);
}
catch (InstantiationException e)
{
throw new YarnRuntimeException(e);
}
}
/// <summary>Return a buffer into the pool.</summary>
/// <remarks>
/// Return a buffer into the pool. After being returned,
/// the buffer may be recycled, so the user must not
/// continue to use it in any way.
/// </remarks>
/// <param name="buf">the buffer to return</param>
public virtual void ReturnBuffer(ByteBuffer buf)
{
buf.Clear();
// reset mark, limit, etc
int size = buf.Capacity();
Queue <WeakReference <ByteBuffer> > list = buffersBySize[size];
if (list == null)
{
list = new ConcurrentLinkedQueue <WeakReference <ByteBuffer> >();
Queue <WeakReference <ByteBuffer> > prev = buffersBySize.PutIfAbsent(size, list);
// someone else put a queue in the map before we did
if (prev != null)
{
list = prev;
}
}
list.AddItem(new WeakReference <ByteBuffer>(buf));
}
internal virtual int Get(T t)
{
if (t == null)
{
return(0);
}
int sn = t2i[t];
if (sn == null)
{
sn = max.GetAndIncrement();
int old = t2i.PutIfAbsent(t, sn);
if (old != null)
{
return(old);
}
i2t[sn] = t;
}
return(sn);
}
/// <summary>
/// Finds the appropriate transition array for the given year.
/// </summary>
/// <param name="year"> the year, not null </param>
/// <returns> the transition array, not null </returns>
private ZoneOffsetTransition[] FindTransitionArray(int year)
{
Integer yearObj = year; // should use Year class, but this saves a class load
ZoneOffsetTransition[] transArray = LastRulesCache[yearObj];
if (transArray != null)
{
return(transArray);
}
ZoneOffsetTransitionRule[] ruleArray = LastRules;
transArray = new ZoneOffsetTransition[ruleArray.Length];
for (int i = 0; i < ruleArray.Length; i++)
{
transArray[i] = ruleArray[i].CreateTransition(year);
}
if (year < LAST_CACHED_YEAR)
{
LastRulesCache.PutIfAbsent(yearObj, transArray);
}
return(transArray);
}
/// <summary>
/// Get a new <code>SharedCacheChecksum</code> object based on the configurable
/// algorithm implementation
/// (see <code>yarn.sharedcache.checksum.algo.impl</code>)
/// </summary>
/// <returns><code>SharedCacheChecksum</code> object</returns>
public static SharedCacheChecksum GetChecksum(Configuration conf)
{
Type clazz = conf.GetClass <SharedCacheChecksum>(YarnConfiguration.SharedCacheChecksumAlgoImpl
, defaultAlgorithm);
SharedCacheChecksum checksum = instances[clazz];
if (checksum == null)
{
try
{
checksum = ReflectionUtils.NewInstance(clazz, conf);
SharedCacheChecksum old = instances.PutIfAbsent(clazz, checksum);
if (old != null)
{
checksum = old;
}
}
catch (Exception e)
{
throw new YarnRuntimeException(e);
}
}
return(checksum);
}
//-----------------------------------------------------------------------
/// <summary>
/// Obtains an instance of {@code ZoneOffset} specifying the total offset in seconds
/// <para>
/// The offset must be in the range {@code -18:00} to {@code +18:00}, which corresponds to -64800 to +64800.
///
/// </para>
/// </summary>
/// <param name="totalSeconds"> the total time-zone offset in seconds, from -64800 to +64800 </param>
/// <returns> the ZoneOffset, not null </returns>
/// <exception cref="DateTimeException"> if the offset is not in the required range </exception>
public static ZoneOffset OfTotalSeconds(int totalSeconds)
{
if (System.Math.Abs(totalSeconds) > MAX_SECONDS)
{
throw new DateTimeException("Zone offset not in valid range: -18:00 to +18:00");
}
if (totalSeconds % (15 * SECONDS_PER_MINUTE) == 0)
{
Integer totalSecs = totalSeconds;
ZoneOffset result = SECONDS_CACHE[totalSecs];
if (result == temporal.TemporalAccessor_Fields.Null)
{
result = new ZoneOffset(totalSeconds);
SECONDS_CACHE.PutIfAbsent(totalSecs, result);
result = SECONDS_CACHE[totalSecs];
ID_CACHE.PutIfAbsent(result.Id, result);
}
return(result);
}
else
{
return(new ZoneOffset(totalSeconds));
}
}
private void InitializeData(Locale desiredLocale)
{
Locale = desiredLocale;
// Copy values of a cached instance if any.
SoftReference <DateFormatSymbols> @ref = CachedInstances[Locale];
DateFormatSymbols dfs;
if (@ref != null && (dfs = @ref.get()) != null)
{
CopyMembers(dfs, this);
return;
}
// Initialize the fields from the ResourceBundle for locale.
LocaleProviderAdapter adapter = LocaleProviderAdapter.getAdapter(typeof(DateFormatSymbolsProvider), Locale);
// Avoid any potential recursions
if (!(adapter is ResourceBundleBasedAdapter))
{
adapter = LocaleProviderAdapter.ResourceBundleBased;
}
ResourceBundle resource = ((ResourceBundleBasedAdapter)adapter).LocaleData.getDateFormatData(Locale);
// JRE and CLDR use different keys
// JRE: Eras, short.Eras and narrow.Eras
// CLDR: long.Eras, Eras and narrow.Eras
if (resource.ContainsKey("Eras"))
{
Eras_Renamed = resource.GetStringArray("Eras");
}
else if (resource.ContainsKey("long.Eras"))
{
Eras_Renamed = resource.GetStringArray("long.Eras");
}
else if (resource.ContainsKey("short.Eras"))
{
Eras_Renamed = resource.GetStringArray("short.Eras");
}
Months_Renamed = resource.GetStringArray("MonthNames");
ShortMonths_Renamed = resource.GetStringArray("MonthAbbreviations");
Ampms = resource.GetStringArray("AmPmMarkers");
LocalPatternChars_Renamed = resource.GetString("DateTimePatternChars");
// Day of week names are stored in a 1-based array.
Weekdays_Renamed = ToOneBasedArray(resource.GetStringArray("DayNames"));
ShortWeekdays_Renamed = ToOneBasedArray(resource.GetStringArray("DayAbbreviations"));
// Put a clone in the cache
@ref = new SoftReference <>((DateFormatSymbols)this.Clone());
SoftReference <DateFormatSymbols> x = CachedInstances.PutIfAbsent(Locale, @ref);
if (x != null)
{
DateFormatSymbols y = x.get();
if (y == null)
{
// Replace the empty SoftReference with ref.
CachedInstances[Locale] = @ref;
}
}
}
public virtual Server GetServer <_T0>(Type protocol, object instance, IPEndPoint addr
, Configuration conf, SecretManager <_T0> secretManager, int numHandlers, string
portRangeConfig)
where _T0 : TokenIdentifier
{
Constructor <object> constructor = serviceCache[protocol];
if (constructor == null)
{
Type pbServiceImplClazz = null;
try
{
pbServiceImplClazz = localConf.GetClassByName(GetPbServiceImplClassName(protocol)
);
}
catch (TypeLoadException e)
{
throw new YarnRuntimeException("Failed to load class: [" + GetPbServiceImplClassName
(protocol) + "]", e);
}
try
{
constructor = pbServiceImplClazz.GetConstructor(protocol);
serviceCache.PutIfAbsent(protocol, constructor);
}
catch (MissingMethodException e)
{
throw new YarnRuntimeException("Could not find constructor with params: " + typeof(
long) + ", " + typeof(IPEndPoint) + ", " + typeof(Configuration), e);
}
}
object service = null;
try
{
service = constructor.NewInstance(instance);
}
catch (TargetInvocationException e)
{
throw new YarnRuntimeException(e);
}
catch (MemberAccessException e)
{
throw new YarnRuntimeException(e);
}
catch (InstantiationException e)
{
throw new YarnRuntimeException(e);
}
Type pbProtocol = service.GetType().GetInterfaces()[0];
MethodInfo method = protoCache[protocol];
if (method == null)
{
Type protoClazz = null;
try
{
protoClazz = localConf.GetClassByName(GetProtoClassName(protocol));
}
catch (TypeLoadException e)
{
throw new YarnRuntimeException("Failed to load class: [" + GetProtoClassName(protocol
) + "]", e);
}
try
{
method = protoClazz.GetMethod("newReflectiveBlockingService", pbProtocol.GetInterfaces
()[0]);
protoCache.PutIfAbsent(protocol, method);
}
catch (MissingMethodException e)
{
throw new YarnRuntimeException(e);
}
}
try
{
return(CreateServer(pbProtocol, addr, conf, secretManager, numHandlers, (BlockingService
)method.Invoke(null, service), portRangeConfig));
}
catch (TargetInvocationException e)
{
throw new YarnRuntimeException(e);
}
catch (MemberAccessException e)
{
throw new YarnRuntimeException(e);
}
catch (IOException e)
{
throw new YarnRuntimeException(e);
}
}