internal virtual int GetNumEntriesInDatabase()
        {
            int             numEntries = 0;
            LeveldbIterator iter       = null;

            try
            {
                iter = new LeveldbIterator(db);
                iter.SeekToFirst();
                while (iter.HasNext())
                {
                    KeyValuePair <byte[], byte[]> entry = iter.Next();
                    Log.Info("entry: " + JniDBFactory.AsString(entry.Key));
                    ++numEntries;
                }
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
            finally
            {
                if (iter != null)
                {
                    iter.Close();
                }
            }
            return(numEntries);
        }
        /// <exception cref="System.IO.IOException"/>
        private int LoadTokens(TimelineStateStore.TimelineServiceState state)
        {
            byte[] @base = LeveldbUtils.KeyBuilder.NewInstance().Add(TokenEntryPrefix).GetBytesForLookup
                               ();
            int             numTokens = 0;
            LeveldbIterator iterator  = null;

            try
            {
                for (iterator = new LeveldbIterator(db), iterator.Seek(@base); iterator.HasNext()
                     ; iterator.Next())
                {
                    byte[] k = iterator.PeekNext().Key;
                    if (!LeveldbUtils.PrefixMatches(@base, @base.Length, k))
                    {
                        break;
                    }
                    byte[] v = iterator.PeekNext().Value;
                    LoadTokenData(state, v);
                    ++numTokens;
                }
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
            finally
            {
                IOUtils.Cleanup(Log, iterator);
            }
            return(numTokens);
        }
Exemple #3
0
        /// <exception cref="System.IO.IOException"/>
        public override NMStateStoreService.RecoveredContainerTokensState LoadContainerTokensState
            ()
        {
            NMStateStoreService.RecoveredContainerTokensState state = new NMStateStoreService.RecoveredContainerTokensState
                                                                          ();
            state.activeTokens = new Dictionary <ContainerId, long>();
            LeveldbIterator iter = null;

            try
            {
                iter = new LeveldbIterator(db);
                iter.Seek(JniDBFactory.Bytes(ContainerTokensKeyPrefix));
                int containerTokensKeyPrefixLength = ContainerTokensKeyPrefix.Length;
                while (iter.HasNext())
                {
                    KeyValuePair <byte[], byte[]> entry = iter.Next();
                    string fullKey = JniDBFactory.AsString(entry.Key);
                    if (!fullKey.StartsWith(ContainerTokensKeyPrefix))
                    {
                        break;
                    }
                    string key = Sharpen.Runtime.Substring(fullKey, containerTokensKeyPrefixLength);
                    if (key.Equals(CurrentMasterKeySuffix))
                    {
                        state.currentMasterKey = ParseMasterKey(entry.Value);
                    }
                    else
                    {
                        if (key.Equals(PrevMasterKeySuffix))
                        {
                            state.previousMasterKey = ParseMasterKey(entry.Value);
                        }
                        else
                        {
                            if (key.StartsWith(ConverterUtils.ContainerPrefix))
                            {
                                LoadContainerToken(state, fullKey, key, entry.Value);
                            }
                        }
                    }
                }
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
            finally
            {
                if (iter != null)
                {
                    iter.Close();
                }
            }
            return(state);
        }
Exemple #4
0
        /// <exception cref="System.IO.IOException"/>
        public override NMStateStoreService.RecoveredApplicationsState LoadApplicationsState
            ()
        {
            NMStateStoreService.RecoveredApplicationsState state = new NMStateStoreService.RecoveredApplicationsState
                                                                       ();
            state.applications = new AList <YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto
                                            >();
            string          keyPrefix = ApplicationsKeyPrefix;
            LeveldbIterator iter      = null;

            try
            {
                iter = new LeveldbIterator(db);
                iter.Seek(JniDBFactory.Bytes(keyPrefix));
                while (iter.HasNext())
                {
                    KeyValuePair <byte[], byte[]> entry = iter.Next();
                    string key = JniDBFactory.AsString(entry.Key);
                    if (!key.StartsWith(keyPrefix))
                    {
                        break;
                    }
                    state.applications.AddItem(YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto
                                               .ParseFrom(entry.Value));
                }
                state.finishedApplications = new AList <ApplicationId>();
                keyPrefix = FinishedAppsKeyPrefix;
                iter.Seek(JniDBFactory.Bytes(keyPrefix));
                while (iter.HasNext())
                {
                    KeyValuePair <byte[], byte[]> entry = iter.Next();
                    string key = JniDBFactory.AsString(entry.Key);
                    if (!key.StartsWith(keyPrefix))
                    {
                        break;
                    }
                    ApplicationId appId = ConverterUtils.ToApplicationId(Sharpen.Runtime.Substring(key
                                                                                                   , keyPrefix.Length));
                    state.finishedApplications.AddItem(appId);
                }
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
            finally
            {
                if (iter != null)
                {
                    iter.Close();
                }
            }
            return(state);
        }
Exemple #5
0
        /// <exception cref="System.IO.IOException"/>
        public override NMStateStoreService.RecoveredLogDeleterState LoadLogDeleterState(
            )
        {
            NMStateStoreService.RecoveredLogDeleterState state = new NMStateStoreService.RecoveredLogDeleterState
                                                                     ();
            state.logDeleterMap = new Dictionary <ApplicationId, YarnServerNodemanagerRecoveryProtos.LogDeleterProto
                                                  >();
            LeveldbIterator iter = null;

            try
            {
                iter = new LeveldbIterator(db);
                iter.Seek(JniDBFactory.Bytes(LogDeleterKeyPrefix));
                int logDeleterKeyPrefixLength = LogDeleterKeyPrefix.Length;
                while (iter.HasNext())
                {
                    KeyValuePair <byte[], byte[]> entry = iter.Next();
                    string fullKey = JniDBFactory.AsString(entry.Key);
                    if (!fullKey.StartsWith(LogDeleterKeyPrefix))
                    {
                        break;
                    }
                    string        appIdStr = Sharpen.Runtime.Substring(fullKey, logDeleterKeyPrefixLength);
                    ApplicationId appId    = null;
                    try
                    {
                        appId = ConverterUtils.ToApplicationId(appIdStr);
                    }
                    catch (ArgumentException)
                    {
                        Log.Warn("Skipping unknown log deleter key " + fullKey);
                        continue;
                    }
                    YarnServerNodemanagerRecoveryProtos.LogDeleterProto proto = YarnServerNodemanagerRecoveryProtos.LogDeleterProto
                                                                                .ParseFrom(entry.Value);
                    state.logDeleterMap[appId] = proto;
                }
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
            finally
            {
                if (iter != null)
                {
                    iter.Close();
                }
            }
            return(state);
        }
Exemple #6
0
        /// <exception cref="System.IO.IOException"/>
        public override NMStateStoreService.RecoveredLocalizationState LoadLocalizationState
            ()
        {
            NMStateStoreService.RecoveredLocalizationState state = new NMStateStoreService.RecoveredLocalizationState
                                                                       ();
            LeveldbIterator iter = null;

            try
            {
                iter = new LeveldbIterator(db);
                iter.Seek(JniDBFactory.Bytes(LocalizationPublicKeyPrefix));
                state.publicTrackerState = LoadResourceTrackerState(iter, LocalizationPublicKeyPrefix
                                                                    );
                iter.Seek(JniDBFactory.Bytes(LocalizationPrivateKeyPrefix));
                while (iter.HasNext())
                {
                    KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
                    string key = JniDBFactory.AsString(entry.Key);
                    if (!key.StartsWith(LocalizationPrivateKeyPrefix))
                    {
                        break;
                    }
                    int userEndPos = key.IndexOf('/', LocalizationPrivateKeyPrefix.Length);
                    if (userEndPos < 0)
                    {
                        throw new IOException("Unable to determine user in resource key: " + key);
                    }
                    string user = Sharpen.Runtime.Substring(key, LocalizationPrivateKeyPrefix.Length,
                                                            userEndPos);
                    state.userResources[user] = LoadUserLocalizedResources(iter, Sharpen.Runtime.Substring
                                                                               (key, 0, userEndPos + 1));
                }
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
            finally
            {
                if (iter != null)
                {
                    iter.Close();
                }
            }
            return(state);
        }
Exemple #7
0
		/// <exception cref="System.IO.IOException"/>
		private int LoadTokenMasterKeys(HistoryServerStateStoreService.HistoryServerState
			 state)
		{
			int numKeys = 0;
			LeveldbIterator iter = null;
			try
			{
				iter = new LeveldbIterator(db);
				iter.Seek(JniDBFactory.Bytes(TokenMasterKeyKeyPrefix));
				while (iter.HasNext())
				{
					KeyValuePair<byte[], byte[]> entry = iter.Next();
					string key = JniDBFactory.AsString(entry.Key);
					if (!key.StartsWith(TokenMasterKeyKeyPrefix))
					{
						break;
					}
					if (Log.IsDebugEnabled())
					{
						Log.Debug("Loading master key from " + key);
					}
					try
					{
						LoadTokenMasterKey(state, entry.Value);
					}
					catch (IOException e)
					{
						throw new IOException("Error loading token master key from " + key, e);
					}
					++numKeys;
				}
			}
			catch (DBException e)
			{
				throw new IOException(e);
			}
			finally
			{
				if (iter != null)
				{
					iter.Close();
				}
			}
			return numKeys;
		}
        /// <exception cref="System.IO.IOException"/>
        private void LoadRMApps(RMStateStore.RMState state)
        {
            int             numApps        = 0;
            int             numAppAttempts = 0;
            LeveldbIterator iter           = null;

            try
            {
                iter = new LeveldbIterator(db);
                iter.Seek(JniDBFactory.Bytes(RmAppKeyPrefix));
                while (iter.HasNext())
                {
                    KeyValuePair <byte[], byte[]> entry = iter.Next();
                    string key = JniDBFactory.AsString(entry.Key);
                    if (!key.StartsWith(RmAppKeyPrefix))
                    {
                        break;
                    }
                    string appIdStr = Sharpen.Runtime.Substring(key, RmAppRoot.Length + 1);
                    if (appIdStr.Contains(Separator))
                    {
                        Log.Warn("Skipping extraneous data " + key);
                        continue;
                    }
                    numAppAttempts += LoadRMApp(state, iter, appIdStr, entry.Value);
                    ++numApps;
                }
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
            finally
            {
                if (iter != null)
                {
                    iter.Close();
                }
            }
            Log.Info("Recovered " + numApps + " applications and " + numAppAttempts + " application attempts"
                     );
        }
        /// <exception cref="System.IO.IOException"/>
        private int LoadRMDTSecretManagerTokens(RMStateStore.RMState state)
        {
            int             numTokens = 0;
            LeveldbIterator iter      = null;

            try
            {
                iter = new LeveldbIterator(db);
                iter.Seek(JniDBFactory.Bytes(RmDtTokenKeyPrefix));
                while (iter.HasNext())
                {
                    KeyValuePair <byte[], byte[]> entry = iter.Next();
                    string key = JniDBFactory.AsString(entry.Key);
                    if (!key.StartsWith(RmDtTokenKeyPrefix))
                    {
                        break;
                    }
                    RMDelegationTokenIdentifierData tokenData = LoadDelegationToken(entry.Value);
                    RMDelegationTokenIdentifier     tokenId   = tokenData.GetTokenIdentifier();
                    long renewDate = tokenData.GetRenewDate();
                    state.rmSecretManagerState.delegationTokenState[tokenId] = renewDate;
                    ++numTokens;
                    if (Log.IsDebugEnabled())
                    {
                        Log.Debug("Loaded RM delegation token from " + key + ": tokenId=" + tokenId + ", renewDate="
                                  + renewDate);
                    }
                }
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
            finally
            {
                if (iter != null)
                {
                    iter.Close();
                }
            }
            return(numTokens);
        }
Exemple #10
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        private bool DeleteNextEntity(string entityType, byte[] ts)
        {
            LeveldbIterator iterator   = null;
            LeveldbIterator pfIterator = null;

            try
            {
                iterator   = ((LeveldbTimelineStore)store).GetDbIterator(false);
                pfIterator = ((LeveldbTimelineStore)store).GetDbIterator(false);
                return(((LeveldbTimelineStore)store).DeleteNextEntity(entityType, ts, iterator, pfIterator
                                                                      , false));
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
            finally
            {
                IOUtils.Cleanup(null, iterator, pfIterator);
            }
        }
        /// <exception cref="System.IO.IOException"/>
        private int LoadRMDTSecretManagerKeys(RMStateStore.RMState state)
        {
            int             numKeys = 0;
            LeveldbIterator iter    = null;

            try
            {
                iter = new LeveldbIterator(db);
                iter.Seek(JniDBFactory.Bytes(RmDtMasterKeyKeyPrefix));
                while (iter.HasNext())
                {
                    KeyValuePair <byte[], byte[]> entry = iter.Next();
                    string key = JniDBFactory.AsString(entry.Key);
                    if (!key.StartsWith(RmDtMasterKeyKeyPrefix))
                    {
                        break;
                    }
                    DelegationKey masterKey = LoadDelegationKey(entry.Value);
                    state.rmSecretManagerState.masterKeyState.AddItem(masterKey);
                    ++numKeys;
                    if (Log.IsDebugEnabled())
                    {
                        Log.Debug("Loaded RM delegation key from " + key + ": keyId=" + masterKey.GetKeyId
                                      () + ", expirationDate=" + masterKey.GetExpiryDate());
                    }
                }
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
            finally
            {
                if (iter != null)
                {
                    iter.Close();
                }
            }
            return(numKeys);
        }
        /// <exception cref="System.IO.IOException"/>
        private int LoadRMApp(RMStateStore.RMState rmState, LeveldbIterator iter, string
                              appIdStr, byte[] appData)
        {
            ApplicationStateData appState = CreateApplicationState(appIdStr, appData);
            ApplicationId        appId    = appState.GetApplicationSubmissionContext().GetApplicationId
                                                ();

            rmState.appState[appId] = appState;
            string attemptNodePrefix = GetApplicationNodeKey(appId) + Separator;

            while (iter.HasNext())
            {
                KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
                string key = JniDBFactory.AsString(entry.Key);
                if (!key.StartsWith(attemptNodePrefix))
                {
                    break;
                }
                string attemptId = Sharpen.Runtime.Substring(key, attemptNodePrefix.Length);
                if (attemptId.StartsWith(ApplicationAttemptId.appAttemptIdStrPrefix))
                {
                    ApplicationAttemptStateData attemptState = CreateAttemptState(attemptId, entry.Value
                                                                                  );
                    appState.attempts[attemptState.GetAttemptId()] = attemptState;
                }
                else
                {
                    Log.Warn("Ignoring unknown application key: " + key);
                }
                iter.Next();
            }
            int numAttempts = appState.attempts.Count;

            if (Log.IsDebugEnabled())
            {
                Log.Debug("Loaded application " + appId + " with " + numAttempts + " attempts");
            }
            return(numAttempts);
        }
Exemple #13
0
        /// <exception cref="System.IO.IOException"/>
        public override NMStateStoreService.RecoveredDeletionServiceState LoadDeletionServiceState
            ()
        {
            NMStateStoreService.RecoveredDeletionServiceState state = new NMStateStoreService.RecoveredDeletionServiceState
                                                                          ();
            state.tasks = new AList <YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto
                                     >();
            LeveldbIterator iter = null;

            try
            {
                iter = new LeveldbIterator(db);
                iter.Seek(JniDBFactory.Bytes(DeletionTaskKeyPrefix));
                while (iter.HasNext())
                {
                    KeyValuePair <byte[], byte[]> entry = iter.Next();
                    string key = JniDBFactory.AsString(entry.Key);
                    if (!key.StartsWith(DeletionTaskKeyPrefix))
                    {
                        break;
                    }
                    state.tasks.AddItem(YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto
                                        .ParseFrom(entry.Value));
                }
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
            finally
            {
                if (iter != null)
                {
                    iter.Close();
                }
            }
            return(state);
        }
Exemple #14
0
        /// <exception cref="System.IO.IOException"/>
        private IList <YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto> LoadCompletedResources
            (LeveldbIterator iter, string keyPrefix)
        {
            IList <YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto> rsrcs = new AList
                                                                                       <YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto>();

            while (iter.HasNext())
            {
                KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
                string key = JniDBFactory.AsString(entry.Key);
                if (!key.StartsWith(keyPrefix))
                {
                    break;
                }
                if (Log.IsDebugEnabled())
                {
                    Log.Debug("Loading completed resource from " + key);
                }
                rsrcs.AddItem(YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto.ParseFrom
                                  (entry.Value));
                iter.Next();
            }
            return(rsrcs);
        }
Exemple #15
0
		/// <exception cref="System.IO.IOException"/>
		private void RecoverState(Configuration conf)
		{
			Path recoveryRoot = GetRecoveryPath();
			if (recoveryRoot != null)
			{
				StartStore(recoveryRoot);
				Sharpen.Pattern jobPattern = Sharpen.Pattern.Compile(JobID.JobidRegex);
				LeveldbIterator iter = null;
				try
				{
					iter = new LeveldbIterator(stateDb);
					iter.Seek(JniDBFactory.Bytes(JobID.Job));
					while (iter.HasNext())
					{
						KeyValuePair<byte[], byte[]> entry = iter.Next();
						string key = JniDBFactory.AsString(entry.Key);
						if (!jobPattern.Matcher(key).Matches())
						{
							break;
						}
						RecoverJobShuffleInfo(key, entry.Value);
					}
				}
				catch (DBException e)
				{
					throw new IOException("Database error during recovery", e);
				}
				finally
				{
					if (iter != null)
					{
						iter.Close();
					}
				}
			}
		}
Exemple #16
0
 /// <exception cref="System.IO.IOException"/>
 private NMStateStoreService.RecoveredContainerState LoadContainerState(ContainerId
                                                                        containerId, LeveldbIterator iter, string keyPrefix)
 {
     NMStateStoreService.RecoveredContainerState rcs = new NMStateStoreService.RecoveredContainerState
                                                           ();
     rcs.status = NMStateStoreService.RecoveredContainerStatus.Requested;
     while (iter.HasNext())
     {
         KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
         string key = JniDBFactory.AsString(entry.Key);
         if (!key.StartsWith(keyPrefix))
         {
             break;
         }
         iter.Next();
         string suffix = Sharpen.Runtime.Substring(key, keyPrefix.Length - 1);
         // start with '/'
         if (suffix.Equals(ContainerRequestKeySuffix))
         {
             rcs.startRequest = new StartContainerRequestPBImpl(YarnServiceProtos.StartContainerRequestProto
                                                                .ParseFrom(entry.Value));
         }
         else
         {
             if (suffix.Equals(ContainerDiagsKeySuffix))
             {
                 rcs.diagnostics = JniDBFactory.AsString(entry.Value);
             }
             else
             {
                 if (suffix.Equals(ContainerLaunchedKeySuffix))
                 {
                     if (rcs.status == NMStateStoreService.RecoveredContainerStatus.Requested)
                     {
                         rcs.status = NMStateStoreService.RecoveredContainerStatus.Launched;
                     }
                 }
                 else
                 {
                     if (suffix.Equals(ContainerKilledKeySuffix))
                     {
                         rcs.killed = true;
                     }
                     else
                     {
                         if (suffix.Equals(ContainerExitCodeKeySuffix))
                         {
                             rcs.status   = NMStateStoreService.RecoveredContainerStatus.Completed;
                             rcs.exitCode = System.Convert.ToInt32(JniDBFactory.AsString(entry.Value));
                         }
                         else
                         {
                             throw new IOException("Unexpected container state key: " + key);
                         }
                     }
                 }
             }
         }
     }
     return(rcs);
 }
Exemple #17
0
        /// <exception cref="System.IO.IOException"/>
        public override NMStateStoreService.RecoveredNMTokensState LoadNMTokensState()
        {
            NMStateStoreService.RecoveredNMTokensState state = new NMStateStoreService.RecoveredNMTokensState
                                                                   ();
            state.applicationMasterKeys = new Dictionary <ApplicationAttemptId, MasterKey>();
            LeveldbIterator iter = null;

            try
            {
                iter = new LeveldbIterator(db);
                iter.Seek(JniDBFactory.Bytes(NmTokensKeyPrefix));
                while (iter.HasNext())
                {
                    KeyValuePair <byte[], byte[]> entry = iter.Next();
                    string fullKey = JniDBFactory.AsString(entry.Key);
                    if (!fullKey.StartsWith(NmTokensKeyPrefix))
                    {
                        break;
                    }
                    string key = Sharpen.Runtime.Substring(fullKey, NmTokensKeyPrefix.Length);
                    if (key.Equals(CurrentMasterKeySuffix))
                    {
                        state.currentMasterKey = ParseMasterKey(entry.Value);
                    }
                    else
                    {
                        if (key.Equals(PrevMasterKeySuffix))
                        {
                            state.previousMasterKey = ParseMasterKey(entry.Value);
                        }
                        else
                        {
                            if (key.StartsWith(ApplicationAttemptId.appAttemptIdStrPrefix))
                            {
                                ApplicationAttemptId attempt;
                                try
                                {
                                    attempt = ConverterUtils.ToApplicationAttemptId(key);
                                }
                                catch (ArgumentException e)
                                {
                                    throw new IOException("Bad application master key state for " + fullKey, e);
                                }
                                state.applicationMasterKeys[attempt] = ParseMasterKey(entry.Value);
                            }
                        }
                    }
                }
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
            finally
            {
                if (iter != null)
                {
                    iter.Close();
                }
            }
            return(state);
        }
Exemple #18
0
 /// <exception cref="System.IO.IOException"/>
 private NMStateStoreService.RecoveredUserResources LoadUserLocalizedResources(LeveldbIterator
                                                                               iter, string keyPrefix)
 {
     NMStateStoreService.RecoveredUserResources userResources = new NMStateStoreService.RecoveredUserResources
                                                                    ();
     while (iter.HasNext())
     {
         KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
         string key = JniDBFactory.AsString(entry.Key);
         if (!key.StartsWith(keyPrefix))
         {
             break;
         }
         if (key.StartsWith(LocalizationFilecacheSuffix, keyPrefix.Length))
         {
             userResources.privateTrackerState = LoadResourceTrackerState(iter, keyPrefix + LocalizationFilecacheSuffix
                                                                          );
         }
         else
         {
             if (key.StartsWith(LocalizationAppcacheSuffix, keyPrefix.Length))
             {
                 int appIdStartPos = keyPrefix.Length + LocalizationAppcacheSuffix.Length;
                 int appIdEndPos   = key.IndexOf('/', appIdStartPos);
                 if (appIdEndPos < 0)
                 {
                     throw new IOException("Unable to determine appID in resource key: " + key);
                 }
                 ApplicationId appId = ConverterUtils.ToApplicationId(Sharpen.Runtime.Substring(key
                                                                                                , appIdStartPos, appIdEndPos));
                 userResources.appTrackerStates[appId] = LoadResourceTrackerState(iter, Sharpen.Runtime.Substring
                                                                                      (key, 0, appIdEndPos + 1));
             }
             else
             {
                 throw new IOException("Unexpected user resource key " + key);
             }
         }
     }
     return(userResources);
 }
Exemple #19
0
        /// <exception cref="System.IO.IOException"/>
        private IDictionary <YarnProtos.LocalResourceProto, Path> LoadStartedResources(LeveldbIterator
                                                                                       iter, string keyPrefix)
        {
            IDictionary <YarnProtos.LocalResourceProto, Path> rsrcs = new Dictionary <YarnProtos.LocalResourceProto
                                                                                      , Path>();

            while (iter.HasNext())
            {
                KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
                string key = JniDBFactory.AsString(entry.Key);
                if (!key.StartsWith(keyPrefix))
                {
                    break;
                }
                Path localPath = new Path(Sharpen.Runtime.Substring(key, keyPrefix.Length));
                if (Log.IsDebugEnabled())
                {
                    Log.Debug("Loading in-progress resource at " + localPath);
                }
                rsrcs[YarnProtos.LocalResourceProto.ParseFrom(entry.Value)] = localPath;
                iter.Next();
            }
            return(rsrcs);
        }
Exemple #20
0
        /// <exception cref="System.IO.IOException"/>
        private NMStateStoreService.LocalResourceTrackerState LoadResourceTrackerState(LeveldbIterator
                                                                                       iter, string keyPrefix)
        {
            string completedPrefix = keyPrefix + LocalizationCompletedSuffix;
            string startedPrefix   = keyPrefix + LocalizationStartedSuffix;

            NMStateStoreService.LocalResourceTrackerState state = new NMStateStoreService.LocalResourceTrackerState
                                                                      ();
            while (iter.HasNext())
            {
                KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
                string key = JniDBFactory.AsString(entry.Key);
                if (!key.StartsWith(keyPrefix))
                {
                    break;
                }
                if (key.StartsWith(completedPrefix))
                {
                    state.localizedResources = LoadCompletedResources(iter, completedPrefix);
                }
                else
                {
                    if (key.StartsWith(startedPrefix))
                    {
                        state.inProgressResources = LoadStartedResources(iter, startedPrefix);
                    }
                    else
                    {
                        throw new IOException("Unexpected key in resource tracker state: " + key);
                    }
                }
            }
            return(state);
        }
Exemple #21
0
        /// <exception cref="System.IO.IOException"/>
        public override IList <NMStateStoreService.RecoveredContainerState> LoadContainersState
            ()
        {
            AList <NMStateStoreService.RecoveredContainerState> containers = new AList <NMStateStoreService.RecoveredContainerState
                                                                                        >();
            AList <ContainerId> containersToRemove = new AList <ContainerId>();
            LeveldbIterator     iter = null;

            try
            {
                iter = new LeveldbIterator(db);
                iter.Seek(JniDBFactory.Bytes(ContainersKeyPrefix));
                while (iter.HasNext())
                {
                    KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
                    string key = JniDBFactory.AsString(entry.Key);
                    if (!key.StartsWith(ContainersKeyPrefix))
                    {
                        break;
                    }
                    int idEndPos = key.IndexOf('/', ContainersKeyPrefix.Length);
                    if (idEndPos < 0)
                    {
                        throw new IOException("Unable to determine container in key: " + key);
                    }
                    ContainerId containerId = ConverterUtils.ToContainerId(Sharpen.Runtime.Substring(
                                                                               key, ContainersKeyPrefix.Length, idEndPos));
                    string keyPrefix = Sharpen.Runtime.Substring(key, 0, idEndPos + 1);
                    NMStateStoreService.RecoveredContainerState rcs = LoadContainerState(containerId,
                                                                                         iter, keyPrefix);
                    // Don't load container without StartContainerRequest
                    if (rcs.startRequest != null)
                    {
                        containers.AddItem(rcs);
                    }
                    else
                    {
                        containersToRemove.AddItem(containerId);
                    }
                }
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
            finally
            {
                if (iter != null)
                {
                    iter.Close();
                }
            }
            // remove container without StartContainerRequest
            foreach (ContainerId containerId_1 in containersToRemove)
            {
                Log.Warn("Remove container " + containerId_1 + " with incomplete records");
                try
                {
                    RemoveContainer(containerId_1);
                }
                catch (IOException e)
                {
                    // TODO: kill and cleanup the leaked container
                    Log.Error("Unable to remove container " + containerId_1 + " in store", e);
                }
            }
            return(containers);
        }