/// <exception cref="System.IO.IOException"/> public override void FinishResourceLocalization(string user, ApplicationId appId, YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto proto) { string localPath = proto.GetLocalPath(); string startedKey = GetResourceStartedKey(user, appId, localPath); string completedKey = GetResourceCompletedKey(user, appId, localPath); if (Log.IsDebugEnabled()) { Log.Debug("Storing localized resource to " + completedKey); } try { WriteBatch batch = db.CreateWriteBatch(); try { batch.Delete(JniDBFactory.Bytes(startedKey)); batch.Put(JniDBFactory.Bytes(completedKey), proto.ToByteArray()); db.Write(batch); } finally { batch.Close(); } } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void RemoveContainer(ContainerId containerId) { string keyPrefix = ContainersKeyPrefix + containerId.ToString(); try { WriteBatch batch = db.CreateWriteBatch(); try { batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerRequestKeySuffix)); batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerDiagsKeySuffix)); batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerLaunchedKeySuffix)); batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerKilledKeySuffix)); batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerExitCodeKeySuffix)); db.Write(batch); } finally { batch.Close(); } } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void RemoveLocalizedResource(string user, ApplicationId appId, Path localPath) { string localPathStr = localPath.ToString(); string startedKey = GetResourceStartedKey(user, appId, localPathStr); string completedKey = GetResourceCompletedKey(user, appId, localPathStr); if (Log.IsDebugEnabled()) { Log.Debug("Removing local resource at " + localPathStr); } try { WriteBatch batch = db.CreateWriteBatch(); try { batch.Delete(JniDBFactory.Bytes(startedKey)); batch.Delete(JniDBFactory.Bytes(completedKey)); db.Write(batch); } finally { batch.Close(); } } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void StoreTokenMasterKey(DelegationKey masterKey) { if (Log.IsDebugEnabled()) { Log.Debug("Storing master key " + masterKey.GetKeyId()); } ByteArrayOutputStream memStream = new ByteArrayOutputStream(); DataOutputStream dataStream = new DataOutputStream(memStream); try { masterKey.Write(dataStream); dataStream.Close(); dataStream = null; } finally { IOUtils.Cleanup(Log, dataStream); } string dbKey = GetTokenMasterKeyDatabaseKey(masterKey); try { db.Put(JniDBFactory.Bytes(dbKey), memStream.ToByteArray()); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void StoreToken(MRDelegationTokenIdentifier tokenId, long renewDate ) { if (Log.IsDebugEnabled()) { Log.Debug("Storing token " + tokenId.GetSequenceNumber()); } ByteArrayOutputStream memStream = new ByteArrayOutputStream(); DataOutputStream dataStream = new DataOutputStream(memStream); try { tokenId.Write(dataStream); dataStream.WriteLong(renewDate); dataStream.Close(); dataStream = null; } finally { IOUtils.Cleanup(Log, dataStream); } string dbKey = GetTokenDatabaseKey(tokenId); try { db.Put(JniDBFactory.Bytes(dbKey), memStream.ToByteArray()); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> protected internal override void StoreRMDTMasterKeyState(DelegationKey masterKey) { string dbKey = GetRMDTMasterKeyNodeKey(masterKey); if (Log.IsDebugEnabled()) { Log.Debug("Storing token master key to " + dbKey); } ByteArrayOutputStream os = new ByteArrayOutputStream(); DataOutputStream @out = new DataOutputStream(os); try { masterKey.Write(@out); } finally { @out.Close(); } try { db.Put(JniDBFactory.Bytes(dbKey), os.ToByteArray()); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> protected internal override void RemoveApplicationStateInternal(ApplicationStateData appState) { ApplicationId appId = appState.GetApplicationSubmissionContext().GetApplicationId (); string appKey = GetApplicationNodeKey(appId); try { WriteBatch batch = db.CreateWriteBatch(); try { batch.Delete(JniDBFactory.Bytes(appKey)); foreach (ApplicationAttemptId attemptId in appState.attempts.Keys) { string attemptKey = GetApplicationAttemptNodeKey(appKey, attemptId); batch.Delete(JniDBFactory.Bytes(attemptKey)); } if (Log.IsDebugEnabled()) { Log.Debug("Removing state for app " + appId + " and " + appState.attempts.Count + " attempts" + " at " + appKey); } db.Write(batch); } finally { batch.Close(); } } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.Exception"/> public override long GetAndIncrementEpoch() { lock (this) { long currentEpoch = 0; byte[] dbKeyBytes = JniDBFactory.Bytes(EpochNode); try { byte[] data = db.Get(dbKeyBytes); if (data != null) { currentEpoch = YarnServerResourceManagerRecoveryProtos.EpochProto.ParseFrom(data) .GetEpoch(); } YarnServerResourceManagerRecoveryProtos.EpochProto proto = Epoch.NewInstance(currentEpoch + 1).GetProto(); db.Put(dbKeyBytes, proto.ToByteArray()); } catch (DBException e) { throw new IOException(e); } return(currentEpoch); } }
protected internal override void StoreOrUpdateAMRMTokenSecretManagerState(AMRMTokenSecretManagerState state, bool isUpdate) { AMRMTokenSecretManagerState data = AMRMTokenSecretManagerState.NewInstance(state); byte[] stateData = data.GetProto().ToByteArray(); db.Put(JniDBFactory.Bytes(AmrmtokenSecretManagerRoot), stateData); }
/// <exception cref="System.IO.IOException"/> public override NMStateStoreService.RecoveredContainerTokensState LoadContainerTokensState () { NMStateStoreService.RecoveredContainerTokensState state = new NMStateStoreService.RecoveredContainerTokensState (); state.activeTokens = new Dictionary <ContainerId, long>(); LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(ContainerTokensKeyPrefix)); int containerTokensKeyPrefixLength = ContainerTokensKeyPrefix.Length; while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.Next(); string fullKey = JniDBFactory.AsString(entry.Key); if (!fullKey.StartsWith(ContainerTokensKeyPrefix)) { break; } string key = Sharpen.Runtime.Substring(fullKey, containerTokensKeyPrefixLength); if (key.Equals(CurrentMasterKeySuffix)) { state.currentMasterKey = ParseMasterKey(entry.Value); } else { if (key.Equals(PrevMasterKeySuffix)) { state.previousMasterKey = ParseMasterKey(entry.Value); } else { if (key.StartsWith(ConverterUtils.ContainerPrefix)) { LoadContainerToken(state, fullKey, key, entry.Value); } } } } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } return(state); }
/// <exception cref="System.IO.IOException"/> public override NMStateStoreService.RecoveredApplicationsState LoadApplicationsState () { NMStateStoreService.RecoveredApplicationsState state = new NMStateStoreService.RecoveredApplicationsState (); state.applications = new AList <YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto >(); string keyPrefix = ApplicationsKeyPrefix; LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(keyPrefix)); while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.Next(); string key = JniDBFactory.AsString(entry.Key); if (!key.StartsWith(keyPrefix)) { break; } state.applications.AddItem(YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto .ParseFrom(entry.Value)); } state.finishedApplications = new AList <ApplicationId>(); keyPrefix = FinishedAppsKeyPrefix; iter.Seek(JniDBFactory.Bytes(keyPrefix)); while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.Next(); string key = JniDBFactory.AsString(entry.Key); if (!key.StartsWith(keyPrefix)) { break; } ApplicationId appId = ConverterUtils.ToApplicationId(Sharpen.Runtime.Substring(key , keyPrefix.Length)); state.finishedApplications.AddItem(appId); } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } return(state); }
internal virtual Version LoadVersion() { byte[] data = stateDb.Get(JniDBFactory.Bytes(StateDbSchemaVersionKey)); // if version is not stored previously, treat it as CURRENT_VERSION_INFO. if (data == null || data.Length == 0) { return GetCurrentVersion(); } Version version = new VersionPBImpl(YarnServerCommonProtos.VersionProto.ParseFrom (data)); return version; }
/// <exception cref="System.IO.IOException"/> internal virtual Version LoadVersion() { byte[] data = db.Get(JniDBFactory.Bytes(DbSchemaVersionKey)); // if version is not stored previously, treat it as 1.0. if (data == null || data.Length == 0) { return Version.NewInstance(1, 0); } Version version = new VersionPBImpl(YarnServerCommonProtos.VersionProto.ParseFrom (data)); return version; }
/// <exception cref="System.IO.IOException"/> public override void RemoveToken(MRDelegationTokenIdentifier tokenId) { string dbKey = GetTokenDatabaseKey(tokenId); try { db.Delete(JniDBFactory.Bytes(dbKey)); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void StoreContainerKilled(ContainerId containerId) { string key = ContainersKeyPrefix + containerId.ToString() + ContainerKilledKeySuffix; try { db.Put(JniDBFactory.Bytes(key), EmptyValue); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void RemoveContainerToken(ContainerId containerId) { string key = ContainerTokensKeyPrefix + containerId; try { db.Delete(JniDBFactory.Bytes(key)); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> internal virtual void DbStoreVersion(Version state) { string key = DbSchemaVersionKey; byte[] data = ((VersionPBImpl)state).GetProto().ToByteArray(); try { db.Put(JniDBFactory.Bytes(key), data); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void RemoveLogDeleter(ApplicationId appId) { string key = GetLogDeleterKey(appId); try { db.Delete(JniDBFactory.Bytes(key)); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> private void StoreSchemaVersion(Version version) { string key = StateDbSchemaVersionKey; byte[] data = ((VersionPBImpl)version).GetProto().ToByteArray(); try { stateDb.Put(JniDBFactory.Bytes(key), data); } catch (DBException e) { throw new IOException(e.Message, e); } }
/// <exception cref="System.IO.IOException"/> private void StoreMasterKey(string dbKey, MasterKey key) { MasterKeyPBImpl pb = (MasterKeyPBImpl)key; try { db.Put(JniDBFactory.Bytes(dbKey), pb.GetProto().ToByteArray()); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void RemoveDeletionTask(int taskId) { string key = DeletionTaskKeyPrefix + taskId; try { db.Delete(JniDBFactory.Bytes(key)); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void StoreFinishedApplication(ApplicationId appId) { string key = FinishedAppsKeyPrefix + appId; try { db.Put(JniDBFactory.Bytes(key), new byte[0]); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void StoreContainerToken(ContainerId containerId, long expTime) { string key = ContainerTokensKeyPrefix + containerId; try { db.Put(JniDBFactory.Bytes(key), JniDBFactory.Bytes(expTime.ToString())); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override NMStateStoreService.RecoveredLogDeleterState LoadLogDeleterState( ) { NMStateStoreService.RecoveredLogDeleterState state = new NMStateStoreService.RecoveredLogDeleterState (); state.logDeleterMap = new Dictionary <ApplicationId, YarnServerNodemanagerRecoveryProtos.LogDeleterProto >(); LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(LogDeleterKeyPrefix)); int logDeleterKeyPrefixLength = LogDeleterKeyPrefix.Length; while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.Next(); string fullKey = JniDBFactory.AsString(entry.Key); if (!fullKey.StartsWith(LogDeleterKeyPrefix)) { break; } string appIdStr = Sharpen.Runtime.Substring(fullKey, logDeleterKeyPrefixLength); ApplicationId appId = null; try { appId = ConverterUtils.ToApplicationId(appIdStr); } catch (ArgumentException) { Log.Warn("Skipping unknown log deleter key " + fullKey); continue; } YarnServerNodemanagerRecoveryProtos.LogDeleterProto proto = YarnServerNodemanagerRecoveryProtos.LogDeleterProto .ParseFrom(entry.Value); state.logDeleterMap[appId] = proto; } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } return(state); }
/// <exception cref="System.IO.IOException"/> public override void StoreApplication(ApplicationId appId, YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto p) { string key = ApplicationsKeyPrefix + appId; try { db.Put(JniDBFactory.Bytes(key), p.ToByteArray()); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void RemoveNMTokenApplicationMasterKey(ApplicationAttemptId attempt ) { string key = NmTokensKeyPrefix + attempt; try { db.Delete(JniDBFactory.Bytes(key)); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void StoreDeletionTask(int taskId, YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto taskProto) { string key = DeletionTaskKeyPrefix + taskId; try { db.Put(JniDBFactory.Bytes(key), taskProto.ToByteArray()); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void StoreLogDeleter(ApplicationId appId, YarnServerNodemanagerRecoveryProtos.LogDeleterProto proto) { string key = GetLogDeleterKey(appId); try { db.Put(JniDBFactory.Bytes(key), proto.ToByteArray()); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void StartResourceLocalization(string user, ApplicationId appId, YarnProtos.LocalResourceProto proto, Path localPath) { string key = GetResourceStartedKey(user, appId, localPath.ToString()); try { db.Put(JniDBFactory.Bytes(key), proto.ToByteArray()); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void StoreContainerDiagnostics(ContainerId containerId, StringBuilder diagnostics) { string key = ContainersKeyPrefix + containerId.ToString() + ContainerDiagsKeySuffix; try { db.Put(JniDBFactory.Bytes(key), JniDBFactory.Bytes(diagnostics.ToString())); } catch (DBException e) { throw new IOException(e); } }