/// <exception cref="System.IO.IOException"/>
public override void RemoveContainer(ContainerId containerId)
{
string keyPrefix = ContainersKeyPrefix + containerId.ToString();
try
{
WriteBatch batch = db.CreateWriteBatch();
try
{
batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerRequestKeySuffix));
batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerDiagsKeySuffix));
batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerLaunchedKeySuffix));
batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerKilledKeySuffix));
batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerExitCodeKeySuffix));
db.Write(batch);
}
finally
{
batch.Close();
}
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public override void FinishResourceLocalization(string user, ApplicationId appId,
YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto proto)
{
string localPath = proto.GetLocalPath();
string startedKey = GetResourceStartedKey(user, appId, localPath);
string completedKey = GetResourceCompletedKey(user, appId, localPath);
if (Log.IsDebugEnabled())
{
Log.Debug("Storing localized resource to " + completedKey);
}
try
{
WriteBatch batch = db.CreateWriteBatch();
try
{
batch.Delete(JniDBFactory.Bytes(startedKey));
batch.Put(JniDBFactory.Bytes(completedKey), proto.ToByteArray());
db.Write(batch);
}
finally
{
batch.Close();
}
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
private NMStateStoreService.LocalResourceTrackerState LoadResourceTrackerState(LeveldbIterator
iter, string keyPrefix)
{
string completedPrefix = keyPrefix + LocalizationCompletedSuffix;
string startedPrefix = keyPrefix + LocalizationStartedSuffix;
NMStateStoreService.LocalResourceTrackerState state = new NMStateStoreService.LocalResourceTrackerState
();
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(keyPrefix))
{
break;
}
if (key.StartsWith(completedPrefix))
{
state.localizedResources = LoadCompletedResources(iter, completedPrefix);
}
else
{
if (key.StartsWith(startedPrefix))
{
state.inProgressResources = LoadStartedResources(iter, startedPrefix);
}
else
{
throw new IOException("Unexpected key in resource tracker state: " + key);
}
}
}
return(state);
}
/// <exception cref="System.IO.IOException"/>
public override void RemoveLocalizedResource(string user, ApplicationId appId, Path
localPath)
{
string localPathStr = localPath.ToString();
string startedKey = GetResourceStartedKey(user, appId, localPathStr);
string completedKey = GetResourceCompletedKey(user, appId, localPathStr);
if (Log.IsDebugEnabled())
{
Log.Debug("Removing local resource at " + localPathStr);
}
try
{
WriteBatch batch = db.CreateWriteBatch();
try
{
batch.Delete(JniDBFactory.Bytes(startedKey));
batch.Delete(JniDBFactory.Bytes(completedKey));
db.Write(batch);
}
finally
{
batch.Close();
}
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public override void StoreTokenMasterKey(DelegationKey masterKey)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Storing master key " + masterKey.GetKeyId());
}
ByteArrayOutputStream memStream = new ByteArrayOutputStream();
DataOutputStream dataStream = new DataOutputStream(memStream);
try
{
masterKey.Write(dataStream);
dataStream.Close();
dataStream = null;
}
finally
{
IOUtils.Cleanup(Log, dataStream);
}
string dbKey = GetTokenMasterKeyDatabaseKey(masterKey);
try
{
db.Put(JniDBFactory.Bytes(dbKey), memStream.ToByteArray());
}
catch (DBException e)
{
throw new IOException(e);
}
}
internal virtual int GetNumEntriesInDatabase()
{
int numEntries = 0;
LeveldbIterator iter = null;
try
{
iter = new LeveldbIterator(db);
iter.SeekToFirst();
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.Next();
Log.Info("entry: " + JniDBFactory.AsString(entry.Key));
++numEntries;
}
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
if (iter != null)
{
iter.Close();
}
}
return(numEntries);
}
/// <exception cref="System.IO.IOException"/>
public override void StoreToken(MRDelegationTokenIdentifier tokenId, long renewDate
)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Storing token " + tokenId.GetSequenceNumber());
}
ByteArrayOutputStream memStream = new ByteArrayOutputStream();
DataOutputStream dataStream = new DataOutputStream(memStream);
try
{
tokenId.Write(dataStream);
dataStream.WriteLong(renewDate);
dataStream.Close();
dataStream = null;
}
finally
{
IOUtils.Cleanup(Log, dataStream);
}
string dbKey = GetTokenDatabaseKey(tokenId);
try
{
db.Put(JniDBFactory.Bytes(dbKey), memStream.ToByteArray());
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
protected internal override void StoreRMDTMasterKeyState(DelegationKey masterKey)
{
string dbKey = GetRMDTMasterKeyNodeKey(masterKey);
if (Log.IsDebugEnabled())
{
Log.Debug("Storing token master key to " + dbKey);
}
ByteArrayOutputStream os = new ByteArrayOutputStream();
DataOutputStream @out = new DataOutputStream(os);
try
{
masterKey.Write(@out);
}
finally
{
@out.Close();
}
try
{
db.Put(JniDBFactory.Bytes(dbKey), os.ToByteArray());
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
protected internal override void RemoveApplicationStateInternal(ApplicationStateData
appState)
{
ApplicationId appId = appState.GetApplicationSubmissionContext().GetApplicationId
();
string appKey = GetApplicationNodeKey(appId);
try
{
WriteBatch batch = db.CreateWriteBatch();
try
{
batch.Delete(JniDBFactory.Bytes(appKey));
foreach (ApplicationAttemptId attemptId in appState.attempts.Keys)
{
string attemptKey = GetApplicationAttemptNodeKey(appKey, attemptId);
batch.Delete(JniDBFactory.Bytes(attemptKey));
}
if (Log.IsDebugEnabled())
{
Log.Debug("Removing state for app " + appId + " and " + appState.attempts.Count +
" attempts" + " at " + appKey);
}
db.Write(batch);
}
finally
{
batch.Close();
}
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.Exception"/>
public override long GetAndIncrementEpoch()
{
lock (this)
{
long currentEpoch = 0;
byte[] dbKeyBytes = JniDBFactory.Bytes(EpochNode);
try
{
byte[] data = db.Get(dbKeyBytes);
if (data != null)
{
currentEpoch = YarnServerResourceManagerRecoveryProtos.EpochProto.ParseFrom(data)
.GetEpoch();
}
YarnServerResourceManagerRecoveryProtos.EpochProto proto = Epoch.NewInstance(currentEpoch
+ 1).GetProto();
db.Put(dbKeyBytes, proto.ToByteArray());
}
catch (DBException e)
{
throw new IOException(e);
}
return(currentEpoch);
}
}
/// <exception cref="System.IO.IOException"/>
protected internal override void StartStorage()
{
Options options = new Options();
Path dbPath = new Path(GetConfig().Get(YarnConfiguration.TimelineServiceLeveldbStateStorePath
), DbName);
FileSystem localFS = null;
try
{
localFS = FileSystem.GetLocal(GetConfig());
if (!localFS.Exists(dbPath))
{
if (!localFS.Mkdirs(dbPath))
{
throw new IOException("Couldn't create directory for leveldb " + "timeline store "
+ dbPath);
}
localFS.SetPermission(dbPath, LeveldbDirUmask);
}
}
finally
{
IOUtils.Cleanup(Log, localFS);
}
JniDBFactory factory = new JniDBFactory();
try
{
options.CreateIfMissing(false);
db = factory.Open(new FilePath(dbPath.ToString()), options);
Log.Info("Loading the existing database at th path: " + dbPath.ToString());
CheckVersion();
}
catch (NativeDB.DBException e)
{
if (e.IsNotFound() || e.Message.Contains(" does not exist "))
{
try
{
options.CreateIfMissing(true);
db = factory.Open(new FilePath(dbPath.ToString()), options);
Log.Info("Creating a new database at th path: " + dbPath.ToString());
StoreVersion(CurrentVersionInfo);
}
catch (DBException ex)
{
throw new IOException(ex);
}
}
else
{
throw new IOException(e);
}
}
catch (DBException e)
{
throw new IOException(e);
}
}
protected internal override void StoreOrUpdateAMRMTokenSecretManagerState(AMRMTokenSecretManagerState
state, bool isUpdate)
{
AMRMTokenSecretManagerState data = AMRMTokenSecretManagerState.NewInstance(state);
byte[] stateData = data.GetProto().ToByteArray();
db.Put(JniDBFactory.Bytes(AmrmtokenSecretManagerRoot), stateData);
}
/// <exception cref="System.IO.IOException"/>
public override NMStateStoreService.RecoveredContainerTokensState LoadContainerTokensState
()
{
NMStateStoreService.RecoveredContainerTokensState state = new NMStateStoreService.RecoveredContainerTokensState
();
state.activeTokens = new Dictionary <ContainerId, long>();
LeveldbIterator iter = null;
try
{
iter = new LeveldbIterator(db);
iter.Seek(JniDBFactory.Bytes(ContainerTokensKeyPrefix));
int containerTokensKeyPrefixLength = ContainerTokensKeyPrefix.Length;
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.Next();
string fullKey = JniDBFactory.AsString(entry.Key);
if (!fullKey.StartsWith(ContainerTokensKeyPrefix))
{
break;
}
string key = Sharpen.Runtime.Substring(fullKey, containerTokensKeyPrefixLength);
if (key.Equals(CurrentMasterKeySuffix))
{
state.currentMasterKey = ParseMasterKey(entry.Value);
}
else
{
if (key.Equals(PrevMasterKeySuffix))
{
state.previousMasterKey = ParseMasterKey(entry.Value);
}
else
{
if (key.StartsWith(ConverterUtils.ContainerPrefix))
{
LoadContainerToken(state, fullKey, key, entry.Value);
}
}
}
}
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
if (iter != null)
{
iter.Close();
}
}
return(state);
}
/// <exception cref="System.IO.IOException"/>
public override NMStateStoreService.RecoveredApplicationsState LoadApplicationsState
()
{
NMStateStoreService.RecoveredApplicationsState state = new NMStateStoreService.RecoveredApplicationsState
();
state.applications = new AList <YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto
>();
string keyPrefix = ApplicationsKeyPrefix;
LeveldbIterator iter = null;
try
{
iter = new LeveldbIterator(db);
iter.Seek(JniDBFactory.Bytes(keyPrefix));
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.Next();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(keyPrefix))
{
break;
}
state.applications.AddItem(YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto
.ParseFrom(entry.Value));
}
state.finishedApplications = new AList <ApplicationId>();
keyPrefix = FinishedAppsKeyPrefix;
iter.Seek(JniDBFactory.Bytes(keyPrefix));
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.Next();
string key = JniDBFactory.AsString(entry.Key);
if (!key.StartsWith(keyPrefix))
{
break;
}
ApplicationId appId = ConverterUtils.ToApplicationId(Sharpen.Runtime.Substring(key
, keyPrefix.Length));
state.finishedApplications.AddItem(appId);
}
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
if (iter != null)
{
iter.Close();
}
}
return(state);
}
/// <exception cref="System.IO.IOException"/>
public override void RemoveToken(MRDelegationTokenIdentifier tokenId)
{
string dbKey = GetTokenDatabaseKey(tokenId);
try
{
db.Delete(JniDBFactory.Bytes(dbKey));
}
catch (DBException e)
{
throw new IOException(e);
}
}
internal virtual Version LoadVersion()
{
byte[] data = stateDb.Get(JniDBFactory.Bytes(StateDbSchemaVersionKey));
// if version is not stored previously, treat it as CURRENT_VERSION_INFO.
if (data == null || data.Length == 0)
{
return GetCurrentVersion();
}
Version version = new VersionPBImpl(YarnServerCommonProtos.VersionProto.ParseFrom
(data));
return version;
}
/// <exception cref="System.IO.IOException"/>
internal virtual Version LoadVersion()
{
byte[] data = db.Get(JniDBFactory.Bytes(DbSchemaVersionKey));
// if version is not stored previously, treat it as 1.0.
if (data == null || data.Length == 0)
{
return Version.NewInstance(1, 0);
}
Version version = new VersionPBImpl(YarnServerCommonProtos.VersionProto.ParseFrom
(data));
return version;
}
/// <exception cref="System.IO.IOException"/>
public override void RemoveContainerToken(ContainerId containerId)
{
string key = ContainerTokensKeyPrefix + containerId;
try
{
db.Delete(JniDBFactory.Bytes(key));
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public override void StoreFinishedApplication(ApplicationId appId)
{
string key = FinishedAppsKeyPrefix + appId;
try
{
db.Put(JniDBFactory.Bytes(key), new byte[0]);
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public override void StoreContainerKilled(ContainerId containerId)
{
string key = ContainersKeyPrefix + containerId.ToString() + ContainerKilledKeySuffix;
try
{
db.Put(JniDBFactory.Bytes(key), EmptyValue);
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public override void RemoveLogDeleter(ApplicationId appId)
{
string key = GetLogDeleterKey(appId);
try
{
db.Delete(JniDBFactory.Bytes(key));
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
private void StoreSchemaVersion(Version version)
{
string key = StateDbSchemaVersionKey;
byte[] data = ((VersionPBImpl)version).GetProto().ToByteArray();
try
{
stateDb.Put(JniDBFactory.Bytes(key), data);
}
catch (DBException e)
{
throw new IOException(e.Message, e);
}
}
/// <exception cref="System.IO.IOException"/>
internal virtual void DbStoreVersion(Version state)
{
string key = DbSchemaVersionKey;
byte[] data = ((VersionPBImpl)state).GetProto().ToByteArray();
try
{
db.Put(JniDBFactory.Bytes(key), data);
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public override void StoreContainerToken(ContainerId containerId, long expTime)
{
string key = ContainerTokensKeyPrefix + containerId;
try
{
db.Put(JniDBFactory.Bytes(key), JniDBFactory.Bytes(expTime.ToString()));
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
private void StoreMasterKey(string dbKey, MasterKey key)
{
MasterKeyPBImpl pb = (MasterKeyPBImpl)key;
try
{
db.Put(JniDBFactory.Bytes(dbKey), pb.GetProto().ToByteArray());
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public override void RemoveDeletionTask(int taskId)
{
string key = DeletionTaskKeyPrefix + taskId;
try
{
db.Delete(JniDBFactory.Bytes(key));
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public override NMStateStoreService.RecoveredLogDeleterState LoadLogDeleterState(
)
{
NMStateStoreService.RecoveredLogDeleterState state = new NMStateStoreService.RecoveredLogDeleterState
();
state.logDeleterMap = new Dictionary <ApplicationId, YarnServerNodemanagerRecoveryProtos.LogDeleterProto
>();
LeveldbIterator iter = null;
try
{
iter = new LeveldbIterator(db);
iter.Seek(JniDBFactory.Bytes(LogDeleterKeyPrefix));
int logDeleterKeyPrefixLength = LogDeleterKeyPrefix.Length;
while (iter.HasNext())
{
KeyValuePair <byte[], byte[]> entry = iter.Next();
string fullKey = JniDBFactory.AsString(entry.Key);
if (!fullKey.StartsWith(LogDeleterKeyPrefix))
{
break;
}
string appIdStr = Sharpen.Runtime.Substring(fullKey, logDeleterKeyPrefixLength);
ApplicationId appId = null;
try
{
appId = ConverterUtils.ToApplicationId(appIdStr);
}
catch (ArgumentException)
{
Log.Warn("Skipping unknown log deleter key " + fullKey);
continue;
}
YarnServerNodemanagerRecoveryProtos.LogDeleterProto proto = YarnServerNodemanagerRecoveryProtos.LogDeleterProto
.ParseFrom(entry.Value);
state.logDeleterMap[appId] = proto;
}
}
catch (DBException e)
{
throw new IOException(e);
}
finally
{
if (iter != null)
{
iter.Close();
}
}
return(state);
}
/// <exception cref="System.IO.IOException"/>
public override void RemoveNMTokenApplicationMasterKey(ApplicationAttemptId attempt
)
{
string key = NmTokensKeyPrefix + attempt;
try
{
db.Delete(JniDBFactory.Bytes(key));
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public override void StoreDeletionTask(int taskId, YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto
taskProto)
{
string key = DeletionTaskKeyPrefix + taskId;
try
{
db.Put(JniDBFactory.Bytes(key), taskProto.ToByteArray());
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
public override void StartResourceLocalization(string user, ApplicationId appId,
YarnProtos.LocalResourceProto proto, Path localPath)
{
string key = GetResourceStartedKey(user, appId, localPath.ToString());
try
{
db.Put(JniDBFactory.Bytes(key), proto.ToByteArray());
}
catch (DBException e)
{
throw new IOException(e);
}
}
