/// <exception cref="System.IO.IOException"/> public override NMStateStoreService.RecoveredApplicationsState LoadApplicationsState () { NMStateStoreService.RecoveredApplicationsState state = new NMStateStoreService.RecoveredApplicationsState (); state.applications = new AList <YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto >(); string keyPrefix = ApplicationsKeyPrefix; LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(keyPrefix)); while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.Next(); string key = JniDBFactory.AsString(entry.Key); if (!key.StartsWith(keyPrefix)) { break; } state.applications.AddItem(YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto .ParseFrom(entry.Value)); } state.finishedApplications = new AList <ApplicationId>(); keyPrefix = FinishedAppsKeyPrefix; iter.Seek(JniDBFactory.Bytes(keyPrefix)); while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.Next(); string key = JniDBFactory.AsString(entry.Key); if (!key.StartsWith(keyPrefix)) { break; } ApplicationId appId = ConverterUtils.ToApplicationId(Sharpen.Runtime.Substring(key , keyPrefix.Length)); state.finishedApplications.AddItem(appId); } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } return(state); }
/// <exception cref="System.IO.IOException"/> private int LoadTokens(TimelineStateStore.TimelineServiceState state) { byte[] @base = LeveldbUtils.KeyBuilder.NewInstance().Add(TokenEntryPrefix).GetBytesForLookup (); int numTokens = 0; LeveldbIterator iterator = null; try { for (iterator = new LeveldbIterator(db), iterator.Seek(@base); iterator.HasNext() ; iterator.Next()) { byte[] k = iterator.PeekNext().Key; if (!LeveldbUtils.PrefixMatches(@base, @base.Length, k)) { break; } byte[] v = iterator.PeekNext().Value; LoadTokenData(state, v); ++numTokens; } } catch (DBException e) { throw new IOException(e); } finally { IOUtils.Cleanup(Log, iterator); } return(numTokens); }
/// <exception cref="System.IO.IOException"/> public override NMStateStoreService.RecoveredLocalizationState LoadLocalizationState () { NMStateStoreService.RecoveredLocalizationState state = new NMStateStoreService.RecoveredLocalizationState (); LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(LocalizationPublicKeyPrefix)); state.publicTrackerState = LoadResourceTrackerState(iter, LocalizationPublicKeyPrefix ); iter.Seek(JniDBFactory.Bytes(LocalizationPrivateKeyPrefix)); while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.PeekNext(); string key = JniDBFactory.AsString(entry.Key); if (!key.StartsWith(LocalizationPrivateKeyPrefix)) { break; } int userEndPos = key.IndexOf('/', LocalizationPrivateKeyPrefix.Length); if (userEndPos < 0) { throw new IOException("Unable to determine user in resource key: " + key); } string user = Sharpen.Runtime.Substring(key, LocalizationPrivateKeyPrefix.Length, userEndPos); state.userResources[user] = LoadUserLocalizedResources(iter, Sharpen.Runtime.Substring (key, 0, userEndPos + 1)); } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } return(state); }
/// <exception cref="System.IO.IOException"/> public override NMStateStoreService.RecoveredContainerTokensState LoadContainerTokensState () { NMStateStoreService.RecoveredContainerTokensState state = new NMStateStoreService.RecoveredContainerTokensState (); state.activeTokens = new Dictionary <ContainerId, long>(); LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(ContainerTokensKeyPrefix)); int containerTokensKeyPrefixLength = ContainerTokensKeyPrefix.Length; while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.Next(); string fullKey = JniDBFactory.AsString(entry.Key); if (!fullKey.StartsWith(ContainerTokensKeyPrefix)) { break; } string key = Sharpen.Runtime.Substring(fullKey, containerTokensKeyPrefixLength); if (key.Equals(CurrentMasterKeySuffix)) { state.currentMasterKey = ParseMasterKey(entry.Value); } else { if (key.Equals(PrevMasterKeySuffix)) { state.previousMasterKey = ParseMasterKey(entry.Value); } else { if (key.StartsWith(ConverterUtils.ContainerPrefix)) { LoadContainerToken(state, fullKey, key, entry.Value); } } } } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } return(state); }
/// <exception cref="System.IO.IOException"/> public override NMStateStoreService.RecoveredLogDeleterState LoadLogDeleterState( ) { NMStateStoreService.RecoveredLogDeleterState state = new NMStateStoreService.RecoveredLogDeleterState (); state.logDeleterMap = new Dictionary <ApplicationId, YarnServerNodemanagerRecoveryProtos.LogDeleterProto >(); LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(LogDeleterKeyPrefix)); int logDeleterKeyPrefixLength = LogDeleterKeyPrefix.Length; while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.Next(); string fullKey = JniDBFactory.AsString(entry.Key); if (!fullKey.StartsWith(LogDeleterKeyPrefix)) { break; } string appIdStr = Sharpen.Runtime.Substring(fullKey, logDeleterKeyPrefixLength); ApplicationId appId = null; try { appId = ConverterUtils.ToApplicationId(appIdStr); } catch (ArgumentException) { Log.Warn("Skipping unknown log deleter key " + fullKey); continue; } YarnServerNodemanagerRecoveryProtos.LogDeleterProto proto = YarnServerNodemanagerRecoveryProtos.LogDeleterProto .ParseFrom(entry.Value); state.logDeleterMap[appId] = proto; } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } return(state); }
/// <exception cref="System.IO.IOException"/> private int LoadTokenMasterKeys(HistoryServerStateStoreService.HistoryServerState state) { int numKeys = 0; LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(TokenMasterKeyKeyPrefix)); while (iter.HasNext()) { KeyValuePair<byte[], byte[]> entry = iter.Next(); string key = JniDBFactory.AsString(entry.Key); if (!key.StartsWith(TokenMasterKeyKeyPrefix)) { break; } if (Log.IsDebugEnabled()) { Log.Debug("Loading master key from " + key); } try { LoadTokenMasterKey(state, entry.Value); } catch (IOException e) { throw new IOException("Error loading token master key from " + key, e); } ++numKeys; } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } return numKeys; }
/// <exception cref="System.IO.IOException"/> private void LoadRMApps(RMStateStore.RMState state) { int numApps = 0; int numAppAttempts = 0; LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(RmAppKeyPrefix)); while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.Next(); string key = JniDBFactory.AsString(entry.Key); if (!key.StartsWith(RmAppKeyPrefix)) { break; } string appIdStr = Sharpen.Runtime.Substring(key, RmAppRoot.Length + 1); if (appIdStr.Contains(Separator)) { Log.Warn("Skipping extraneous data " + key); continue; } numAppAttempts += LoadRMApp(state, iter, appIdStr, entry.Value); ++numApps; } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } Log.Info("Recovered " + numApps + " applications and " + numAppAttempts + " application attempts" ); }
/// <exception cref="System.IO.IOException"/> private int LoadRMDTSecretManagerTokens(RMStateStore.RMState state) { int numTokens = 0; LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(RmDtTokenKeyPrefix)); while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.Next(); string key = JniDBFactory.AsString(entry.Key); if (!key.StartsWith(RmDtTokenKeyPrefix)) { break; } RMDelegationTokenIdentifierData tokenData = LoadDelegationToken(entry.Value); RMDelegationTokenIdentifier tokenId = tokenData.GetTokenIdentifier(); long renewDate = tokenData.GetRenewDate(); state.rmSecretManagerState.delegationTokenState[tokenId] = renewDate; ++numTokens; if (Log.IsDebugEnabled()) { Log.Debug("Loaded RM delegation token from " + key + ": tokenId=" + tokenId + ", renewDate=" + renewDate); } } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } return(numTokens); }
/// <exception cref="System.IO.IOException"/> private int LoadRMDTSecretManagerKeys(RMStateStore.RMState state) { int numKeys = 0; LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(RmDtMasterKeyKeyPrefix)); while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.Next(); string key = JniDBFactory.AsString(entry.Key); if (!key.StartsWith(RmDtMasterKeyKeyPrefix)) { break; } DelegationKey masterKey = LoadDelegationKey(entry.Value); state.rmSecretManagerState.masterKeyState.AddItem(masterKey); ++numKeys; if (Log.IsDebugEnabled()) { Log.Debug("Loaded RM delegation key from " + key + ": keyId=" + masterKey.GetKeyId () + ", expirationDate=" + masterKey.GetExpiryDate()); } } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } return(numKeys); }
/// <exception cref="System.IO.IOException"/> public override NMStateStoreService.RecoveredDeletionServiceState LoadDeletionServiceState () { NMStateStoreService.RecoveredDeletionServiceState state = new NMStateStoreService.RecoveredDeletionServiceState (); state.tasks = new AList <YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto >(); LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(DeletionTaskKeyPrefix)); while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.Next(); string key = JniDBFactory.AsString(entry.Key); if (!key.StartsWith(DeletionTaskKeyPrefix)) { break; } state.tasks.AddItem(YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto .ParseFrom(entry.Value)); } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } return(state); }
/// <exception cref="System.IO.IOException"/> private void RecoverState(Configuration conf) { Path recoveryRoot = GetRecoveryPath(); if (recoveryRoot != null) { StartStore(recoveryRoot); Sharpen.Pattern jobPattern = Sharpen.Pattern.Compile(JobID.JobidRegex); LeveldbIterator iter = null; try { iter = new LeveldbIterator(stateDb); iter.Seek(JniDBFactory.Bytes(JobID.Job)); while (iter.HasNext()) { KeyValuePair<byte[], byte[]> entry = iter.Next(); string key = JniDBFactory.AsString(entry.Key); if (!jobPattern.Matcher(key).Matches()) { break; } RecoverJobShuffleInfo(key, entry.Value); } } catch (DBException e) { throw new IOException("Database error during recovery", e); } finally { if (iter != null) { iter.Close(); } } } }
/// <exception cref="System.IO.IOException"/> public override NMStateStoreService.RecoveredNMTokensState LoadNMTokensState() { NMStateStoreService.RecoveredNMTokensState state = new NMStateStoreService.RecoveredNMTokensState (); state.applicationMasterKeys = new Dictionary <ApplicationAttemptId, MasterKey>(); LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(NmTokensKeyPrefix)); while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.Next(); string fullKey = JniDBFactory.AsString(entry.Key); if (!fullKey.StartsWith(NmTokensKeyPrefix)) { break; } string key = Sharpen.Runtime.Substring(fullKey, NmTokensKeyPrefix.Length); if (key.Equals(CurrentMasterKeySuffix)) { state.currentMasterKey = ParseMasterKey(entry.Value); } else { if (key.Equals(PrevMasterKeySuffix)) { state.previousMasterKey = ParseMasterKey(entry.Value); } else { if (key.StartsWith(ApplicationAttemptId.appAttemptIdStrPrefix)) { ApplicationAttemptId attempt; try { attempt = ConverterUtils.ToApplicationAttemptId(key); } catch (ArgumentException e) { throw new IOException("Bad application master key state for " + fullKey, e); } state.applicationMasterKeys[attempt] = ParseMasterKey(entry.Value); } } } } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } return(state); }
/// <exception cref="System.IO.IOException"/> public override IList <NMStateStoreService.RecoveredContainerState> LoadContainersState () { AList <NMStateStoreService.RecoveredContainerState> containers = new AList <NMStateStoreService.RecoveredContainerState >(); AList <ContainerId> containersToRemove = new AList <ContainerId>(); LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(ContainersKeyPrefix)); while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.PeekNext(); string key = JniDBFactory.AsString(entry.Key); if (!key.StartsWith(ContainersKeyPrefix)) { break; } int idEndPos = key.IndexOf('/', ContainersKeyPrefix.Length); if (idEndPos < 0) { throw new IOException("Unable to determine container in key: " + key); } ContainerId containerId = ConverterUtils.ToContainerId(Sharpen.Runtime.Substring( key, ContainersKeyPrefix.Length, idEndPos)); string keyPrefix = Sharpen.Runtime.Substring(key, 0, idEndPos + 1); NMStateStoreService.RecoveredContainerState rcs = LoadContainerState(containerId, iter, keyPrefix); // Don't load container without StartContainerRequest if (rcs.startRequest != null) { containers.AddItem(rcs); } else { containersToRemove.AddItem(containerId); } } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } // remove container without StartContainerRequest foreach (ContainerId containerId_1 in containersToRemove) { Log.Warn("Remove container " + containerId_1 + " with incomplete records"); try { RemoveContainer(containerId_1); } catch (IOException e) { // TODO: kill and cleanup the leaked container Log.Error("Unable to remove container " + containerId_1 + " in store", e); } } return(containers); }