/// <exception cref="System.IO.IOException"/>
private int LoadTokens(TimelineStateStore.TimelineServiceState state)
{
byte[] @base = LeveldbUtils.KeyBuilder.NewInstance().Add(TokenEntryPrefix).GetBytesForLookup
();
int numTokens = 0;
LeveldbIterator iterator = null;
try
{
for (iterator = new LeveldbIterator(db), iterator.Seek(@base); iterator.HasNext()
; iterator.Next())
{
byte[] k = iterator.PeekNext().Key;
if (!LeveldbUtils.PrefixMatches(@base, @base.Length, k))
{
break;
}
byte[] v = iterator.PeekNext().Value;
LoadTokenData(state, v);
++numTokens;
}
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
IOUtils.Cleanup(Log, iterator);
}
return(numTokens);
}
/// <exception cref="System.IO.IOException"/>
private NMStateStoreService.LocalResourceTrackerState LoadResourceTrackerState(LeveldbIterator
iter, string keyPrefix)
{
string completedPrefix = keyPrefix + LocalizationCompletedSuffix;
string startedPrefix = keyPrefix + LocalizationStartedSuffix;
NMStateStoreService.LocalResourceTrackerState state = new NMStateStoreService.LocalResourceTrackerState
();
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(keyPrefix))
{
break;
}
if (key.StartsWith(completedPrefix))
{
state.localizedResources = LoadCompletedResources(iter, completedPrefix);
}
else
{
if (key.StartsWith(startedPrefix))
{
state.inProgressResources = LoadStartedResources(iter, startedPrefix);
}
else
{
throw new IOException("Unexpected key in resource tracker state: " + key);
}
}
}
return(state);
}
/// <exception cref="System.IO.IOException"/>
public override NMStateStoreService.RecoveredLocalizationState LoadLocalizationState
()
{
NMStateStoreService.RecoveredLocalizationState state = new NMStateStoreService.RecoveredLocalizationState
();
LeveldbIterator iter = null;
try
{
iter = new LeveldbIterator(db);
iter.Seek(JniDBFactory.Bytes(LocalizationPublicKeyPrefix));
state.publicTrackerState = LoadResourceTrackerState(iter, LocalizationPublicKeyPrefix
);
iter.Seek(JniDBFactory.Bytes(LocalizationPrivateKeyPrefix));
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(LocalizationPrivateKeyPrefix))
{
break;
}
int userEndPos = key.IndexOf('/', LocalizationPrivateKeyPrefix.Length);
if (userEndPos < 0)
{
throw new IOException("Unable to determine user in resource key: " + key);
}
string user = Sharpen.Runtime.Substring(key, LocalizationPrivateKeyPrefix.Length,
userEndPos);
state.userResources[user] = LoadUserLocalizedResources(iter, Sharpen.Runtime.Substring
(key, 0, userEndPos + 1));
}
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
if (iter != null)
{
iter.Close();
}
}
return(state);
}
/// <exception cref="System.IO.IOException"/>
private NMStateStoreService.RecoveredUserResources LoadUserLocalizedResources(LeveldbIterator
iter, string keyPrefix)
{
NMStateStoreService.RecoveredUserResources userResources = new NMStateStoreService.RecoveredUserResources
();
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(keyPrefix))
{
break;
}
if (key.StartsWith(LocalizationFilecacheSuffix, keyPrefix.Length))
{
userResources.privateTrackerState = LoadResourceTrackerState(iter, keyPrefix + LocalizationFilecacheSuffix
);
}
else
{
if (key.StartsWith(LocalizationAppcacheSuffix, keyPrefix.Length))
{
int appIdStartPos = keyPrefix.Length + LocalizationAppcacheSuffix.Length;
int appIdEndPos = key.IndexOf('/', appIdStartPos);
if (appIdEndPos < 0)
{
throw new IOException("Unable to determine appID in resource key: " + key);
}
ApplicationId appId = ConverterUtils.ToApplicationId(Sharpen.Runtime.Substring(key
, appIdStartPos, appIdEndPos));
userResources.appTrackerStates[appId] = LoadResourceTrackerState(iter, Sharpen.Runtime.Substring
(key, 0, appIdEndPos + 1));
}
else
{
throw new IOException("Unexpected user resource key " + key);
}
}
}
return(userResources);
}
/// <exception cref="System.IO.IOException"/>
private int LoadRMApp(RMStateStore.RMState rmState, LeveldbIterator iter, string
appIdStr, byte[] appData)
{
ApplicationStateData appState = CreateApplicationState(appIdStr, appData);
ApplicationId appId = appState.GetApplicationSubmissionContext().GetApplicationId
();
rmState.appState[appId] = appState;
string attemptNodePrefix = GetApplicationNodeKey(appId) + Separator;
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(attemptNodePrefix))
{
break;
}
string attemptId = Sharpen.Runtime.Substring(key, attemptNodePrefix.Length);
if (attemptId.StartsWith(ApplicationAttemptId.appAttemptIdStrPrefix))
{
ApplicationAttemptStateData attemptState = CreateAttemptState(attemptId, entry.Value
);
appState.attempts[attemptState.GetAttemptId()] = attemptState;
}
else
{
Log.Warn("Ignoring unknown application key: " + key);
}
iter.Next();
}
int numAttempts = appState.attempts.Count;
if (Log.IsDebugEnabled())
{
Log.Debug("Loaded application " + appId + " with " + numAttempts + " attempts");
}
return(numAttempts);
}
/// <exception cref="System.IO.IOException"/>
private IDictionary <YarnProtos.LocalResourceProto, Path> LoadStartedResources(LeveldbIterator
iter, string keyPrefix)
{
IDictionary <YarnProtos.LocalResourceProto, Path> rsrcs = new Dictionary <YarnProtos.LocalResourceProto
, Path>();
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(keyPrefix))
{
break;
}
Path localPath = new Path(Sharpen.Runtime.Substring(key, keyPrefix.Length));
if (Log.IsDebugEnabled())
{
Log.Debug("Loading in-progress resource at " + localPath);
}
rsrcs[YarnProtos.LocalResourceProto.ParseFrom(entry.Value)] = localPath;
iter.Next();
}
return(rsrcs);
}
/// <exception cref="System.IO.IOException"/>
private IList <YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto> LoadCompletedResources
(LeveldbIterator iter, string keyPrefix)
{
IList <YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto> rsrcs = new AList
<YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto>();
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(keyPrefix))
{
break;
}
if (Log.IsDebugEnabled())
{
Log.Debug("Loading completed resource from " + key);
}
rsrcs.AddItem(YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto.ParseFrom
(entry.Value));
iter.Next();
}
return(rsrcs);
}
/// <exception cref="System.IO.IOException"/>
private NMStateStoreService.RecoveredContainerState LoadContainerState(ContainerId
containerId, LeveldbIterator iter, string keyPrefix)
{
NMStateStoreService.RecoveredContainerState rcs = new NMStateStoreService.RecoveredContainerState
();
rcs.status = NMStateStoreService.RecoveredContainerStatus.Requested;
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(keyPrefix))
{
break;
}
iter.Next();
string suffix = Sharpen.Runtime.Substring(key, keyPrefix.Length - 1);
// start with '/'
if (suffix.Equals(ContainerRequestKeySuffix))
{
rcs.startRequest = new StartContainerRequestPBImpl(YarnServiceProtos.StartContainerRequestProto
.ParseFrom(entry.Value));
}
else
{
if (suffix.Equals(ContainerDiagsKeySuffix))
{
rcs.diagnostics = JniDBFactory.AsString(entry.Value);
}
else
{
if (suffix.Equals(ContainerLaunchedKeySuffix))
{
if (rcs.status == NMStateStoreService.RecoveredContainerStatus.Requested)
{
rcs.status = NMStateStoreService.RecoveredContainerStatus.Launched;
}
}
else
{
if (suffix.Equals(ContainerKilledKeySuffix))
{
rcs.killed = true;
}
else
{
if (suffix.Equals(ContainerExitCodeKeySuffix))
{
rcs.status = NMStateStoreService.RecoveredContainerStatus.Completed;
rcs.exitCode = System.Convert.ToInt32(JniDBFactory.AsString(entry.Value));
}
else
{
throw new IOException("Unexpected container state key: " + key);
}
}
}
}
}
}
return(rcs);
}
/// <exception cref="System.IO.IOException"/>
public override IList <NMStateStoreService.RecoveredContainerState> LoadContainersState
()
{
AList <NMStateStoreService.RecoveredContainerState> containers = new AList <NMStateStoreService.RecoveredContainerState
>();
AList <ContainerId> containersToRemove = new AList <ContainerId>();
LeveldbIterator iter = null;
try
{
iter = new LeveldbIterator(db);
iter.Seek(JniDBFactory.Bytes(ContainersKeyPrefix));
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(ContainersKeyPrefix))
{
break;
}
int idEndPos = key.IndexOf('/', ContainersKeyPrefix.Length);
if (idEndPos < 0)
{
throw new IOException("Unable to determine container in key: " + key);
}
ContainerId containerId = ConverterUtils.ToContainerId(Sharpen.Runtime.Substring(
key, ContainersKeyPrefix.Length, idEndPos));
string keyPrefix = Sharpen.Runtime.Substring(key, 0, idEndPos + 1);
NMStateStoreService.RecoveredContainerState rcs = LoadContainerState(containerId,
iter, keyPrefix);
// Don't load container without StartContainerRequest
if (rcs.startRequest != null)
{
containers.AddItem(rcs);
}
else
{
containersToRemove.AddItem(containerId);
}
}
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
if (iter != null)
{
iter.Close();
}
}
// remove container without StartContainerRequest
foreach (ContainerId containerId_1 in containersToRemove)
{
Log.Warn("Remove container " + containerId_1 + " with incomplete records");
try
{
RemoveContainer(containerId_1);
}
catch (IOException e)
{
// TODO: kill and cleanup the leaked container
Log.Error("Unable to remove container " + containerId_1 + " in store", e);
}
}
return(containers);
}
